2024-11-21 03:23:53,505 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 03:23:53,530 main DEBUG Took 0.021273 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 03:23:53,531 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 03:23:53,532 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 03:23:53,533 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 03:23:53,535 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,544 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 03:23:53,569 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,571 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,572 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,573 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,574 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,574 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,576 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,577 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,578 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,578 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,581 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,582 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,582 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,584 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,585 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,586 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,588 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,590 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,591 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:23:53,593 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,593 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 03:23:53,596 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:23:53,598 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 03:23:53,602 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 03:23:53,603 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 03:23:53,605 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 03:23:53,606 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 03:23:53,620 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 03:23:53,624 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 03:23:53,627 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 03:23:53,627 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 03:23:53,628 main DEBUG createAppenders(={Console}) 2024-11-21 03:23:53,629 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-21 03:23:53,630 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 03:23:53,630 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-21 03:23:53,633 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 03:23:53,634 main DEBUG OutputStream closed 2024-11-21 03:23:53,634 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 03:23:53,634 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 03:23:53,635 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-21 03:23:53,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 03:23:53,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 03:23:53,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 03:23:53,782 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 03:23:53,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 03:23:53,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 03:23:53,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 03:23:53,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 03:23:53,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 03:23:53,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 03:23:53,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 03:23:53,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 03:23:53,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 03:23:53,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 03:23:53,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 03:23:53,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 03:23:53,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 03:23:53,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 03:23:53,792 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 03:23:53,793 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-21 03:23:53,793 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 03:23:53,794 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-21T03:23:54,140 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00 2024-11-21 03:23:54,145 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 03:23:54,146 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T03:23:54,160 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-21T03:23:54,196 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T03:23:54,220 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/cluster_0b1aaf4b-5e34-6dbd-0179-21f4d40d50a7, deleteOnExit=true 2024-11-21T03:23:54,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T03:23:54,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/test.cache.data in system properties and HBase conf 2024-11-21T03:23:54,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T03:23:54,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/hadoop.log.dir in system properties and HBase conf 2024-11-21T03:23:54,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T03:23:54,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T03:23:54,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T03:23:54,332 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T03:23:54,432 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:23:54,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T03:23:54,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T03:23:54,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T03:23:54,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T03:23:54,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T03:23:54,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T03:23:54,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T03:23:54,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T03:23:54,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T03:23:54,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/nfs.dump.dir in system properties and HBase conf 2024-11-21T03:23:54,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/java.io.tmpdir in system properties and HBase conf 2024-11-21T03:23:54,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T03:23:54,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T03:23:54,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T03:23:56,484 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T03:23:56,562 INFO [Time-limited test {}] log.Log(170): Logging initialized @4266ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T03:23:56,639 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T03:23:56,700 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T03:23:56,723 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T03:23:56,723 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T03:23:56,725 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T03:23:56,737 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T03:23:56,741 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/hadoop.log.dir/,AVAILABLE} 2024-11-21T03:23:56,742 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T03:23:56,913 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/java.io.tmpdir/jetty-localhost-35739-hadoop-hdfs-3_4_1-tests_jar-_-any-8439995261990315210/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T03:23:56,920 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35739} 2024-11-21T03:23:56,920 INFO [Time-limited test {}] server.Server(415): Started @4625ms 2024-11-21T03:23:57,583 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T03:23:57,595 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T03:23:57,604 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T03:23:57,604 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T03:23:57,604 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T03:23:57,606 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/hadoop.log.dir/,AVAILABLE} 2024-11-21T03:23:57,607 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T03:23:57,722 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/java.io.tmpdir/jetty-localhost-35999-hadoop-hdfs-3_4_1-tests_jar-_-any-12586494837513602901/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T03:23:57,723 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:35999} 2024-11-21T03:23:57,723 INFO [Time-limited test {}] server.Server(415): Started @5428ms 2024-11-21T03:23:57,793 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T03:23:59,183 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/cluster_0b1aaf4b-5e34-6dbd-0179-21f4d40d50a7/data/data1/current/BP-547451667-172.17.0.2-1732159435428/current, will proceed with Du for space computation calculation, 2024-11-21T03:23:59,183 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/cluster_0b1aaf4b-5e34-6dbd-0179-21f4d40d50a7/data/data2/current/BP-547451667-172.17.0.2-1732159435428/current, will proceed with Du for space computation calculation, 2024-11-21T03:23:59,231 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T03:23:59,296 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf3f1e51a81277d31 with lease ID 0xca04ce6ad6ad7c4c: Processing first storage report for DS-456f41fb-3aa2-442d-a074-7697f0ce9de3 from datanode DatanodeRegistration(127.0.0.1:44993, datanodeUuid=338cfed7-1fb1-4a89-94ce-02f4b7325128, infoPort=35005, infoSecurePort=0, ipcPort=35805, storageInfo=lv=-57;cid=testClusterID;nsid=1124774234;c=1732159435428) 2024-11-21T03:23:59,298 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf3f1e51a81277d31 with lease ID 0xca04ce6ad6ad7c4c: from storage DS-456f41fb-3aa2-442d-a074-7697f0ce9de3 node DatanodeRegistration(127.0.0.1:44993, datanodeUuid=338cfed7-1fb1-4a89-94ce-02f4b7325128, infoPort=35005, infoSecurePort=0, ipcPort=35805, storageInfo=lv=-57;cid=testClusterID;nsid=1124774234;c=1732159435428), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-21T03:23:59,299 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf3f1e51a81277d31 with lease ID 0xca04ce6ad6ad7c4c: Processing first storage report for DS-fc9da92d-31c5-4833-b58d-c03eea144357 from datanode DatanodeRegistration(127.0.0.1:44993, datanodeUuid=338cfed7-1fb1-4a89-94ce-02f4b7325128, infoPort=35005, infoSecurePort=0, ipcPort=35805, storageInfo=lv=-57;cid=testClusterID;nsid=1124774234;c=1732159435428) 2024-11-21T03:23:59,300 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf3f1e51a81277d31 with lease ID 0xca04ce6ad6ad7c4c: from storage DS-fc9da92d-31c5-4833-b58d-c03eea144357 node DatanodeRegistration(127.0.0.1:44993, datanodeUuid=338cfed7-1fb1-4a89-94ce-02f4b7325128, infoPort=35005, infoSecurePort=0, ipcPort=35805, storageInfo=lv=-57;cid=testClusterID;nsid=1124774234;c=1732159435428), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T03:23:59,320 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00 2024-11-21T03:23:59,418 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/cluster_0b1aaf4b-5e34-6dbd-0179-21f4d40d50a7/zookeeper_0, clientPort=61758, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/cluster_0b1aaf4b-5e34-6dbd-0179-21f4d40d50a7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/cluster_0b1aaf4b-5e34-6dbd-0179-21f4d40d50a7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T03:23:59,431 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61758 2024-11-21T03:23:59,447 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:23:59,451 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:23:59,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741825_1001 (size=7) 2024-11-21T03:24:00,132 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1 with version=8 2024-11-21T03:24:00,132 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/hbase-staging 2024-11-21T03:24:00,222 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T03:24:00,661 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b69e2a1ee061:0 server-side Connection retries=45 2024-11-21T03:24:00,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:24:00,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:24:00,678 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:24:00,678 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:24:00,678 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:24:00,923 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T03:24:01,026 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T03:24:01,039 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T03:24:01,046 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:24:01,082 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 30855 (auto-detected) 2024-11-21T03:24:01,084 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T03:24:01,109 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42023 2024-11-21T03:24:01,138 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42023 connecting to ZooKeeper ensemble=127.0.0.1:61758 2024-11-21T03:24:01,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:420230x0, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:24:01,242 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42023-0x1015b6b7dcc0000 connected 2024-11-21T03:24:01,378 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:24:01,381 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:24:01,393 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:24:01,398 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1, hbase.cluster.distributed=false 2024-11-21T03:24:01,429 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:24:01,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42023 2024-11-21T03:24:01,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42023 2024-11-21T03:24:01,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42023 2024-11-21T03:24:01,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42023 2024-11-21T03:24:01,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42023 2024-11-21T03:24:01,586 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b69e2a1ee061:0 server-side Connection retries=45 2024-11-21T03:24:01,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:24:01,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:24:01,589 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:24:01,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:24:01,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:24:01,592 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T03:24:01,596 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:24:01,597 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45649 2024-11-21T03:24:01,599 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45649 connecting to ZooKeeper ensemble=127.0.0.1:61758 2024-11-21T03:24:01,601 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:24:01,605 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:24:01,649 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:456490x0, quorum=127.0.0.1:61758, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:24:01,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456490x0, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:24:01,655 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T03:24:01,668 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45649-0x1015b6b7dcc0001 connected 2024-11-21T03:24:01,690 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T03:24:01,695 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T03:24:01,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:24:01,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45649 2024-11-21T03:24:01,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45649 2024-11-21T03:24:01,729 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45649 2024-11-21T03:24:01,739 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45649 2024-11-21T03:24:01,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45649 2024-11-21T03:24:01,766 DEBUG [M:0;b69e2a1ee061:42023 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b69e2a1ee061:42023 2024-11-21T03:24:01,768 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:01,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:24:01,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:24:01,789 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:01,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:01,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T03:24:01,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:01,837 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T03:24:01,839 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b69e2a1ee061,42023,1732159440464 from backup master directory 2024-11-21T03:24:01,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:01,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:24:01,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:24:01,858 WARN [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:24:01,858 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:01,861 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T03:24:01,863 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T03:24:01,954 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/hbase.id] with ID: acaff57a-38ea-4fc8-a8f9-17336e0f736f 2024-11-21T03:24:01,954 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/.tmp/hbase.id 2024-11-21T03:24:01,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741826_1002 (size=42) 2024-11-21T03:24:01,991 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/.tmp/hbase.id]:[hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/hbase.id] 2024-11-21T03:24:02,102 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:24:02,111 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T03:24:02,153 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 33ms. 2024-11-21T03:24:02,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:02,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:02,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741827_1003 (size=196) 2024-11-21T03:24:02,262 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:24:02,265 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T03:24:02,291 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:02,296 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:24:02,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741828_1004 (size=1189) 2024-11-21T03:24:02,350 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store 2024-11-21T03:24:02,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741829_1005 (size=34) 2024-11-21T03:24:02,379 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T03:24:02,382 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:24:02,384 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T03:24:02,384 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:24:02,384 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:24:02,387 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T03:24:02,387 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:24:02,387 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:24:02,389 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732159442384Disabling compacts and flushes for region at 1732159442384Disabling writes for close at 1732159442387 (+3 ms)Writing region close event to WAL at 1732159442387Closed at 1732159442387 2024-11-21T03:24:02,392 WARN [master/b69e2a1ee061:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/.initializing 2024-11-21T03:24:02,392 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/WALs/b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:02,404 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:24:02,424 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b69e2a1ee061%2C42023%2C1732159440464, suffix=, logDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/WALs/b69e2a1ee061,42023,1732159440464, archiveDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/oldWALs, maxLogs=10 2024-11-21T03:24:02,448 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/WALs/b69e2a1ee061,42023,1732159440464/b69e2a1ee061%2C42023%2C1732159440464.1732159442429, exclude list is [], retry=0 2024-11-21T03:24:02,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44993,DS-456f41fb-3aa2-442d-a074-7697f0ce9de3,DISK] 2024-11-21T03:24:02,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T03:24:02,518 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/WALs/b69e2a1ee061,42023,1732159440464/b69e2a1ee061%2C42023%2C1732159440464.1732159442429 2024-11-21T03:24:02,519 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35005:35005)] 2024-11-21T03:24:02,520 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:24:02,521 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:24:02,525 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,527 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,585 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T03:24:02,636 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:02,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:24:02,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T03:24:02,649 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:02,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:02,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T03:24:02,656 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:02,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:02,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T03:24:02,662 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:02,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:02,664 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,668 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,670 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,676 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,678 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,684 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T03:24:02,690 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:24:02,696 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:24:02,698 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62960417, jitterRate=-0.06181667745113373}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T03:24:02,707 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732159442544Initializing all the Stores at 1732159442547 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732159442547Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159442548 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159442548Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159442548Cleaning up temporary data from old regions at 1732159442678 (+130 ms)Region opened successfully at 1732159442707 (+29 ms) 2024-11-21T03:24:02,708 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T03:24:02,751 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d03a7d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b69e2a1ee061/172.17.0.2:0 2024-11-21T03:24:02,786 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T03:24:02,801 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T03:24:02,802 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T03:24:02,805 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T03:24:02,806 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T03:24:02,811 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T03:24:02,811 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T03:24:02,837 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T03:24:02,845 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T03:24:02,859 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T03:24:02,862 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T03:24:02,865 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T03:24:02,875 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T03:24:02,879 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T03:24:02,884 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T03:24:02,896 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T03:24:02,898 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T03:24:02,906 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T03:24:02,926 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T03:24:02,938 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T03:24:02,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T03:24:02,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T03:24:02,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:02,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:02,957 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b69e2a1ee061,42023,1732159440464, sessionid=0x1015b6b7dcc0000, setting cluster-up flag (Was=false) 2024-11-21T03:24:02,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:02,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:03,164 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T03:24:03,167 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:03,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:03,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:03,333 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T03:24:03,335 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:03,341 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T03:24:03,351 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(746): ClusterId : acaff57a-38ea-4fc8-a8f9-17336e0f736f 2024-11-21T03:24:03,354 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T03:24:03,367 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T03:24:03,367 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T03:24:03,382 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T03:24:03,383 DEBUG [RS:0;b69e2a1ee061:45649 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ebfa0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b69e2a1ee061/172.17.0.2:0 2024-11-21T03:24:03,401 DEBUG [RS:0;b69e2a1ee061:45649 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b69e2a1ee061:45649 2024-11-21T03:24:03,404 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T03:24:03,404 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T03:24:03,404 DEBUG [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T03:24:03,406 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(2659): reportForDuty to master=b69e2a1ee061,42023,1732159440464 with port=45649, startcode=1732159441548 2024-11-21T03:24:03,417 DEBUG [RS:0;b69e2a1ee061:45649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T03:24:03,421 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T03:24:03,431 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T03:24:03,439 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T03:24:03,447 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60267, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T03:24:03,447 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b69e2a1ee061,42023,1732159440464 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T03:24:03,452 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42023 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:03,455 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b69e2a1ee061:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:24:03,456 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b69e2a1ee061:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:24:03,456 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b69e2a1ee061:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:24:03,456 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b69e2a1ee061:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:24:03,457 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b69e2a1ee061:0, corePoolSize=10, maxPoolSize=10 2024-11-21T03:24:03,457 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,457 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b69e2a1ee061:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:24:03,457 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,459 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732159473459 2024-11-21T03:24:03,461 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T03:24:03,462 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T03:24:03,463 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:24:03,464 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T03:24:03,466 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T03:24:03,466 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T03:24:03,466 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T03:24:03,467 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T03:24:03,470 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:03,470 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T03:24:03,470 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,474 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T03:24:03,475 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T03:24:03,476 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T03:24:03,484 DEBUG [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T03:24:03,484 WARN [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T03:24:03,485 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T03:24:03,485 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T03:24:03,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741831_1007 (size=1321) 2024-11-21T03:24:03,487 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b69e2a1ee061:0:becomeActiveMaster-HFileCleaner.large.0-1732159443487,5,FailOnTimeoutGroup] 2024-11-21T03:24:03,489 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b69e2a1ee061:0:becomeActiveMaster-HFileCleaner.small.0-1732159443488,5,FailOnTimeoutGroup] 2024-11-21T03:24:03,490 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,490 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T03:24:03,490 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T03:24:03,491 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1 2024-11-21T03:24:03,491 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,492 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741832_1008 (size=32) 2024-11-21T03:24:03,586 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(2659): reportForDuty to master=b69e2a1ee061,42023,1732159440464 with port=45649, startcode=1732159441548 2024-11-21T03:24:03,588 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42023 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:03,591 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42023 {}] master.ServerManager(517): Registering regionserver=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:03,602 DEBUG [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1 2024-11-21T03:24:03,603 DEBUG [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41459 2024-11-21T03:24:03,603 DEBUG [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T03:24:03,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T03:24:03,628 DEBUG [RS:0;b69e2a1ee061:45649 {}] zookeeper.ZKUtil(111): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:03,628 WARN [RS:0;b69e2a1ee061:45649 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:24:03,628 INFO [RS:0;b69e2a1ee061:45649 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:24:03,629 DEBUG [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/WALs/b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:03,631 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b69e2a1ee061,45649,1732159441548] 2024-11-21T03:24:03,657 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T03:24:03,677 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T03:24:03,682 INFO [RS:0;b69e2a1ee061:45649 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T03:24:03,682 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,684 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T03:24:03,690 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T03:24:03,692 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,692 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,693 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,693 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,693 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,694 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,694 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b69e2a1ee061:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:24:03,694 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,694 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,695 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,695 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,695 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,695 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b69e2a1ee061:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:24:03,696 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b69e2a1ee061:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:24:03,696 DEBUG [RS:0;b69e2a1ee061:45649 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:24:03,700 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,701 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,701 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,701 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,702 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,702 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=b69e2a1ee061,45649,1732159441548-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:24:03,733 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T03:24:03,735 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=b69e2a1ee061,45649,1732159441548-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,736 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,736 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.Replication(171): b69e2a1ee061,45649,1732159441548 started 2024-11-21T03:24:03,765 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:03,765 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(1482): Serving as b69e2a1ee061,45649,1732159441548, RpcServer on b69e2a1ee061/172.17.0.2:45649, sessionid=0x1015b6b7dcc0001 2024-11-21T03:24:03,766 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T03:24:03,767 DEBUG [RS:0;b69e2a1ee061:45649 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:03,767 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b69e2a1ee061,45649,1732159441548' 2024-11-21T03:24:03,767 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T03:24:03,769 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T03:24:03,770 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T03:24:03,771 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T03:24:03,771 DEBUG [RS:0;b69e2a1ee061:45649 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:03,771 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b69e2a1ee061,45649,1732159441548' 2024-11-21T03:24:03,771 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T03:24:03,773 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T03:24:03,774 DEBUG [RS:0;b69e2a1ee061:45649 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T03:24:03,774 INFO [RS:0;b69e2a1ee061:45649 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T03:24:03,774 INFO [RS:0;b69e2a1ee061:45649 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T03:24:03,880 INFO [RS:0;b69e2a1ee061:45649 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:24:03,884 INFO [RS:0;b69e2a1ee061:45649 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b69e2a1ee061%2C45649%2C1732159441548, suffix=, logDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/WALs/b69e2a1ee061,45649,1732159441548, archiveDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/oldWALs, maxLogs=32 2024-11-21T03:24:03,909 DEBUG [RS:0;b69e2a1ee061:45649 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/WALs/b69e2a1ee061,45649,1732159441548/b69e2a1ee061%2C45649%2C1732159441548.1732159443887, exclude list is [], retry=0 2024-11-21T03:24:03,909 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:24:03,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T03:24:03,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44993,DS-456f41fb-3aa2-442d-a074-7697f0ce9de3,DISK] 2024-11-21T03:24:03,920 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T03:24:03,920 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:03,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:24:03,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T03:24:03,924 INFO [RS:0;b69e2a1ee061:45649 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/WALs/b69e2a1ee061,45649,1732159441548/b69e2a1ee061%2C45649%2C1732159441548.1732159443887 2024-11-21T03:24:03,926 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T03:24:03,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:03,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:24:03,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T03:24:03,928 DEBUG [RS:0;b69e2a1ee061:45649 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35005:35005)] 2024-11-21T03:24:03,932 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T03:24:03,932 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:03,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:24:03,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T03:24:03,940 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T03:24:03,940 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:03,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:24:03,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T03:24:03,944 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740 2024-11-21T03:24:03,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740 2024-11-21T03:24:03,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T03:24:03,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T03:24:03,951 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:24:03,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T03:24:03,960 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:24:03,961 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71996127, jitterRate=0.07282589375972748}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:24:03,965 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732159443910Initializing all the Stores at 1732159443912 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732159443912Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732159443913 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159443913Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732159443913Cleaning up temporary data from old regions at 1732159443950 (+37 ms)Region opened successfully at 1732159443965 (+15 ms) 2024-11-21T03:24:03,965 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T03:24:03,966 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T03:24:03,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T03:24:03,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T03:24:03,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T03:24:03,968 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T03:24:03,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732159443965Disabling compacts and flushes for region at 1732159443965Disabling writes for close at 1732159443966 (+1 ms)Writing region close event to WAL at 1732159443967 (+1 ms)Closed at 1732159443968 (+1 ms) 2024-11-21T03:24:03,972 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:24:03,973 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T03:24:03,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T03:24:03,988 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T03:24:03,991 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T03:24:04,145 DEBUG [b69e2a1ee061:42023 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T03:24:04,158 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:04,167 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b69e2a1ee061,45649,1732159441548, state=OPENING 2024-11-21T03:24:04,207 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T03:24:04,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:04,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:24:04,251 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:24:04,251 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:24:04,254 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T03:24:04,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:24:04,431 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T03:24:04,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41685, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T03:24:04,447 INFO [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T03:24:04,448 INFO [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:24:04,448 INFO [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T03:24:04,453 INFO [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b69e2a1ee061%2C45649%2C1732159441548.meta, suffix=.meta, logDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/WALs/b69e2a1ee061,45649,1732159441548, archiveDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/oldWALs, maxLogs=32 2024-11-21T03:24:04,469 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/WALs/b69e2a1ee061,45649,1732159441548/b69e2a1ee061%2C45649%2C1732159441548.meta.1732159444455.meta, exclude list is [], retry=0 2024-11-21T03:24:04,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44993,DS-456f41fb-3aa2-442d-a074-7697f0ce9de3,DISK] 2024-11-21T03:24:04,477 INFO [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/WALs/b69e2a1ee061,45649,1732159441548/b69e2a1ee061%2C45649%2C1732159441548.meta.1732159444455.meta 2024-11-21T03:24:04,479 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35005:35005)] 2024-11-21T03:24:04,479 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:24:04,481 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T03:24:04,483 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T03:24:04,489 INFO [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T03:24:04,495 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T03:24:04,496 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:24:04,497 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T03:24:04,497 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T03:24:04,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T03:24:04,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T03:24:04,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:04,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:24:04,504 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T03:24:04,505 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T03:24:04,505 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:04,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:24:04,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T03:24:04,508 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T03:24:04,509 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:04,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:24:04,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T03:24:04,512 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T03:24:04,512 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:04,513 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:24:04,513 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T03:24:04,515 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740 2024-11-21T03:24:04,518 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740 2024-11-21T03:24:04,521 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T03:24:04,521 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T03:24:04,522 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:24:04,526 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T03:24:04,528 INFO [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66508620, jitterRate=-0.008944332599639893}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:24:04,529 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T03:24:04,531 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732159444498Writing region info on filesystem at 1732159444498Initializing all the Stores at 1732159444500 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732159444500Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732159444500Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159444500Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732159444500Cleaning up temporary data from old regions at 1732159444522 (+22 ms)Running coprocessor post-open hooks at 1732159444529 (+7 ms)Region opened successfully at 1732159444531 (+2 ms) 2024-11-21T03:24:04,539 INFO [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732159444424 2024-11-21T03:24:04,552 DEBUG [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T03:24:04,553 INFO [RS_OPEN_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T03:24:04,555 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:04,559 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b69e2a1ee061,45649,1732159441548, state=OPEN 2024-11-21T03:24:04,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:24:04,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:24:04,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:24:04,608 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:04,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:24:04,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T03:24:04,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b69e2a1ee061,45649,1732159441548 in 351 msec 2024-11-21T03:24:04,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T03:24:04,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 640 msec 2024-11-21T03:24:04,628 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:24:04,628 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T03:24:04,655 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:04,657 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:04,688 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:04,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39475, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:04,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3490 sec 2024-11-21T03:24:04,726 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732159444726, completionTime=-1 2024-11-21T03:24:04,729 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-21T03:24:04,730 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-21T03:24:04,770 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-21T03:24:04,770 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732159504770 2024-11-21T03:24:04,771 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732159564770 2024-11-21T03:24:04,771 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 41 msec 2024-11-21T03:24:04,774 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b69e2a1ee061,42023,1732159440464-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:04,774 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b69e2a1ee061,42023,1732159440464-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:04,775 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b69e2a1ee061,42023,1732159440464-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:04,777 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b69e2a1ee061:42023, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:04,777 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:04,778 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:04,787 DEBUG [master/b69e2a1ee061:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T03:24:04,816 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.957sec 2024-11-21T03:24:04,818 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T03:24:04,819 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T03:24:04,821 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T03:24:04,822 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T03:24:04,822 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T03:24:04,823 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b69e2a1ee061,42023,1732159440464-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:24:04,823 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b69e2a1ee061,42023,1732159440464-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T03:24:04,865 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T03:24:04,867 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T03:24:04,867 INFO [master/b69e2a1ee061:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b69e2a1ee061,42023,1732159440464-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:24:04,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:04,891 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T03:24:04,891 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T03:24:04,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:04,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:04,911 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ab58c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:04,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:04,936 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:04,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:04,941 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:04,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:04,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:04,998 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=638, ProcessCount=11, AvailableMemoryMB=3337 2024-11-21T03:24:05,010 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:24:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78a466cf 2024-11-21T03:24:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:24:05,024 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:24:05,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:24:05,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:24:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:05,044 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:24:05,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-21T03:24:05,047 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:05,050 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:24:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:24:05,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741835_1011 (size=959) 2024-11-21T03:24:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:24:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:24:05,467 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1 2024-11-21T03:24:05,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741836_1012 (size=53) 2024-11-21T03:24:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:24:05,880 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:24:05,881 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing aebdff1855164d839f0828203eeecd04, disabling compactions & flushes 2024-11-21T03:24:05,881 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:05,881 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:05,881 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. after waiting 0 ms 2024-11-21T03:24:05,881 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:05,881 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:05,881 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for aebdff1855164d839f0828203eeecd04: Waiting for close lock at 1732159445881Disabling compacts and flushes for region at 1732159445881Disabling writes for close at 1732159445881Writing region close event to WAL at 1732159445881Closed at 1732159445881 2024-11-21T03:24:05,884 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:24:05,889 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732159445884"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732159445884"}]},"ts":"1732159445884"} 2024-11-21T03:24:05,894 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:24:05,897 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:24:05,899 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159445897"}]},"ts":"1732159445897"} 2024-11-21T03:24:05,904 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:24:05,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aebdff1855164d839f0828203eeecd04, ASSIGN}] 2024-11-21T03:24:05,909 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aebdff1855164d839f0828203eeecd04, ASSIGN 2024-11-21T03:24:05,911 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aebdff1855164d839f0828203eeecd04, ASSIGN; state=OFFLINE, location=b69e2a1ee061,45649,1732159441548; forceNewPlan=false, retain=false 2024-11-21T03:24:06,063 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=aebdff1855164d839f0828203eeecd04, regionState=OPENING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:06,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aebdff1855164d839f0828203eeecd04, ASSIGN because future has completed 2024-11-21T03:24:06,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:24:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:24:06,230 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:06,230 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:24:06,230 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,231 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:24:06,231 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,231 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,233 INFO [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,235 INFO [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aebdff1855164d839f0828203eeecd04 columnFamilyName A 2024-11-21T03:24:06,235 DEBUG [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:06,237 INFO [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] regionserver.HStore(327): Store=aebdff1855164d839f0828203eeecd04/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:06,237 INFO [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,239 INFO [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aebdff1855164d839f0828203eeecd04 columnFamilyName B 2024-11-21T03:24:06,239 DEBUG [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:06,240 INFO [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] regionserver.HStore(327): Store=aebdff1855164d839f0828203eeecd04/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:06,240 INFO [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,242 INFO [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aebdff1855164d839f0828203eeecd04 columnFamilyName C 2024-11-21T03:24:06,242 DEBUG [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:06,243 INFO [StoreOpener-aebdff1855164d839f0828203eeecd04-1 {}] regionserver.HStore(327): Store=aebdff1855164d839f0828203eeecd04/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:06,243 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,245 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,245 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,247 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,248 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,248 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:24:06,251 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,255 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:24:06,256 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened aebdff1855164d839f0828203eeecd04; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62683715, jitterRate=-0.06593985855579376}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:24:06,256 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:06,258 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for aebdff1855164d839f0828203eeecd04: Running coprocessor pre-open hook at 1732159446231Writing region info on filesystem at 1732159446231Initializing all the Stores at 1732159446233 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159446233Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159446233Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159446233Cleaning up temporary data from old regions at 1732159446248 (+15 ms)Running coprocessor post-open hooks at 1732159446256 (+8 ms)Region opened successfully at 1732159446258 (+2 ms) 2024-11-21T03:24:06,260 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., pid=6, masterSystemTime=1732159446224 2024-11-21T03:24:06,264 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:06,264 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:06,265 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=aebdff1855164d839f0828203eeecd04, regionState=OPEN, openSeqNum=2, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:06,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:24:06,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T03:24:06,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 in 203 msec 2024-11-21T03:24:06,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T03:24:06,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aebdff1855164d839f0828203eeecd04, ASSIGN in 370 msec 2024-11-21T03:24:06,284 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:24:06,284 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159446284"}]},"ts":"1732159446284"} 2024-11-21T03:24:06,287 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:24:06,289 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:24:06,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2530 sec 2024-11-21T03:24:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:24:07,209 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:07,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:07,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:07,216 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c6058d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:07,220 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:07,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e5dcf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,222 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:07,226 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:07,229 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:24:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:07,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10ab891d 2024-11-21T03:24:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:24:07,235 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:24:07,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:07,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:07,241 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2886558d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:07,244 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4e6a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:07,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:07,248 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:07,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:07,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7290076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:07,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:07,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:07,255 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3687d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:07,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:07,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:07,263 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:07,266 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ede423c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:07,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:07,269 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a044d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:07,272 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@418d3001, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:07,276 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:07,277 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201fc254, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:07,280 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7273d8b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:07,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:07,283 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ee4412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:07,286 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3b3fdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,288 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:07,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:07,289 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6e851d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:07,291 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29434e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:07,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:07,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:07,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:07,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:07,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56225fcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:07,298 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:07,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7b74c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:07,304 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:07,304 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:07,306 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:07,306 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:07,306 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:07,308 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:07,308 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:07,306 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:07,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:07,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:07,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:07,310 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:07,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:07,311 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:07,312 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:07,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:07,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:07,316 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:24:07,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:24:07,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:07,315 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:24:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:24:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:24:07,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:07,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:24:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:24:07,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:24:07,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:24:07,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-21T03:24:07,348 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:24:07,351 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:07,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:07,438 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:07,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:24:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:24:07,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T03:24:07,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:07,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:07,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:07,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:07,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:07,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/67cf36c192964c0385926183733b7154 is 50, key is test_row_0/A:col10/1732159447420/Put/seqid=0 2024-11-21T03:24:07,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741837_1013 (size=12001) 2024-11-21T03:24:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159507644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/67cf36c192964c0385926183733b7154 2024-11-21T03:24:07,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159507650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159507651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:24:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159507657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159507657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T03:24:07,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:07,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:07,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:07,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:07,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:07,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159507770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159507771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159507771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159507789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159507789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/bac35f613e60460889fcea94304eeff0 is 50, key is test_row_0/B:col10/1732159447420/Put/seqid=0 2024-11-21T03:24:07,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741838_1014 (size=12001) 2024-11-21T03:24:07,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T03:24:07,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:07,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:07,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:07,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:07,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:07,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:24:07,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159507989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159507992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:07,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159507992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:07,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:07,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159508000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159508000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T03:24:08,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:08,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:08,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:08,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:08,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:08,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T03:24:08,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:08,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:08,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:08,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:08,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:08,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/bac35f613e60460889fcea94304eeff0 2024-11-21T03:24:08,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159508302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/6ea237037f32450dafa22193dc02857b is 50, key is test_row_0/C:col10/1732159447420/Put/seqid=0 2024-11-21T03:24:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159508309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159508310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159508318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159508319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741839_1015 (size=12001) 2024-11-21T03:24:08,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/6ea237037f32450dafa22193dc02857b 2024-11-21T03:24:08,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T03:24:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:08,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:08,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:08,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:08,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/67cf36c192964c0385926183733b7154 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/67cf36c192964c0385926183733b7154 2024-11-21T03:24:08,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/67cf36c192964c0385926183733b7154, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T03:24:08,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/bac35f613e60460889fcea94304eeff0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/bac35f613e60460889fcea94304eeff0 2024-11-21T03:24:08,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/bac35f613e60460889fcea94304eeff0, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T03:24:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/6ea237037f32450dafa22193dc02857b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/6ea237037f32450dafa22193dc02857b 2024-11-21T03:24:08,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/6ea237037f32450dafa22193dc02857b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T03:24:08,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for aebdff1855164d839f0828203eeecd04 in 1044ms, sequenceid=12, compaction requested=false 2024-11-21T03:24:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:24:08,496 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T03:24:08,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:08,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T03:24:08,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:08,554 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:24:08,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/853126dcf673489980680fed8925142d is 50, key is test_row_0/A:col10/1732159447643/Put/seqid=0 2024-11-21T03:24:08,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741840_1016 (size=12001) 2024-11-21T03:24:08,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:08,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159508851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159508856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159508857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159508858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159508866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159508968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:08,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159508978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159508991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159508993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159508993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,037 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/853126dcf673489980680fed8925142d 2024-11-21T03:24:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/64224194ccf34b3d86cf6c2f0e9ee9e7 is 50, key is test_row_0/B:col10/1732159447643/Put/seqid=0 2024-11-21T03:24:09,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741841_1017 (size=12001) 2024-11-21T03:24:09,122 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/64224194ccf34b3d86cf6c2f0e9ee9e7 2024-11-21T03:24:09,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/a2011bd035204fa2bb860397e441284d is 50, key is test_row_0/C:col10/1732159447643/Put/seqid=0 2024-11-21T03:24:09,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741842_1018 (size=12001) 2024-11-21T03:24:09,190 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/a2011bd035204fa2bb860397e441284d 2024-11-21T03:24:09,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159509188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159509198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159509208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159509210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/853126dcf673489980680fed8925142d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/853126dcf673489980680fed8925142d 2024-11-21T03:24:09,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159509209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,234 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/853126dcf673489980680fed8925142d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:24:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/64224194ccf34b3d86cf6c2f0e9ee9e7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/64224194ccf34b3d86cf6c2f0e9ee9e7 2024-11-21T03:24:09,266 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/64224194ccf34b3d86cf6c2f0e9ee9e7, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:24:09,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/a2011bd035204fa2bb860397e441284d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a2011bd035204fa2bb860397e441284d 2024-11-21T03:24:09,288 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a2011bd035204fa2bb860397e441284d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:24:09,291 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for aebdff1855164d839f0828203eeecd04 in 737ms, sequenceid=37, compaction requested=false 2024-11-21T03:24:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:09,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:09,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-21T03:24:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-21T03:24:09,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T03:24:09,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9430 sec 2024-11-21T03:24:09,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9670 sec 2024-11-21T03:24:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:24:09,509 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:09,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-21T03:24:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:09,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:24:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:24:09,518 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:09,520 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:09,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:09,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/5db2bda6ec9841c0b09e891177422ec6 is 50, key is test_row_0/A:col10/1732159448855/Put/seqid=0 2024-11-21T03:24:09,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741843_1019 (size=14341) 2024-11-21T03:24:09,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159509597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159509602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159509604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:24:09,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159509604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159509614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:24:09,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:09,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:09,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:09,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159509729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159509729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159509739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159509740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159509740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:24:09,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:24:09,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:09,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:09,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:09,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:09,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:09,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159509942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159509942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159509948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159509960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159509960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:09,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/5db2bda6ec9841c0b09e891177422ec6 2024-11-21T03:24:10,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/efac21f7460c4248a9f5e51ef1921199 is 50, key is test_row_0/B:col10/1732159448855/Put/seqid=0 2024-11-21T03:24:10,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:24:10,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:10,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,021 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:24:10,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741844_1020 (size=12001) 2024-11-21T03:24:10,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/efac21f7460c4248a9f5e51ef1921199 2024-11-21T03:24:10,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/137a6a94db554fc19d5ba0ace55b5e90 is 50, key is test_row_0/C:col10/1732159448855/Put/seqid=0 2024-11-21T03:24:10,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741845_1021 (size=12001) 2024-11-21T03:24:10,138 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T03:24:10,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/137a6a94db554fc19d5ba0ace55b5e90 2024-11-21T03:24:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:24:10,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/5db2bda6ec9841c0b09e891177422ec6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/5db2bda6ec9841c0b09e891177422ec6 2024-11-21T03:24:10,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:24:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/5db2bda6ec9841c0b09e891177422ec6, entries=200, sequenceid=49, filesize=14.0 K 2024-11-21T03:24:10,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/efac21f7460c4248a9f5e51ef1921199 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/efac21f7460c4248a9f5e51ef1921199 2024-11-21T03:24:10,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/efac21f7460c4248a9f5e51ef1921199, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T03:24:10,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/137a6a94db554fc19d5ba0ace55b5e90 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/137a6a94db554fc19d5ba0ace55b5e90 2024-11-21T03:24:10,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/137a6a94db554fc19d5ba0ace55b5e90, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T03:24:10,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for aebdff1855164d839f0828203eeecd04 in 740ms, sequenceid=49, compaction requested=true 2024-11-21T03:24:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:10,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:10,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:10,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:10,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:10,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:10,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:10,265 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:10,266 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:10,273 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:10,274 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:10,275 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/A is initiating minor compaction (all files) 2024-11-21T03:24:10,275 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/B is initiating minor compaction (all files) 2024-11-21T03:24:10,276 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/A in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,277 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/67cf36c192964c0385926183733b7154, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/853126dcf673489980680fed8925142d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/5db2bda6ec9841c0b09e891177422ec6] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=37.4 K 2024-11-21T03:24:10,277 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/B in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,278 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/bac35f613e60460889fcea94304eeff0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/64224194ccf34b3d86cf6c2f0e9ee9e7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/efac21f7460c4248a9f5e51ef1921199] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=35.2 K 2024-11-21T03:24:10,281 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:10,281 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting bac35f613e60460889fcea94304eeff0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732159447333 2024-11-21T03:24:10,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:24:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:10,282 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67cf36c192964c0385926183733b7154, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732159447333 2024-11-21T03:24:10,284 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 853126dcf673489980680fed8925142d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732159447634 2024-11-21T03:24:10,286 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 64224194ccf34b3d86cf6c2f0e9ee9e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732159447634 2024-11-21T03:24:10,287 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5db2bda6ec9841c0b09e891177422ec6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732159448849 2024-11-21T03:24:10,289 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting efac21f7460c4248a9f5e51ef1921199, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732159448849 2024-11-21T03:24:10,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/2df9596da1f3451b80cbe0af83aa4ca4 is 50, key is test_row_0/A:col10/1732159450276/Put/seqid=0 2024-11-21T03:24:10,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741846_1022 (size=12001) 2024-11-21T03:24:10,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/2df9596da1f3451b80cbe0af83aa4ca4 2024-11-21T03:24:10,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:24:10,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:10,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159510314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159510311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159510317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159510329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159510321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,357 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#B#compaction#10 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:10,357 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#A#compaction#11 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:10,358 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/f0fd693f5877443fbf1abaf2792a78ef is 50, key is test_row_0/B:col10/1732159448855/Put/seqid=0 2024-11-21T03:24:10,358 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/c537c84544204758ad3703f4ceb3a914 is 50, key is test_row_0/A:col10/1732159448855/Put/seqid=0 2024-11-21T03:24:10,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/7f0bfb8f692148139682554069242fcd is 50, key is test_row_0/B:col10/1732159450276/Put/seqid=0 2024-11-21T03:24:10,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741848_1024 (size=12104) 2024-11-21T03:24:10,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741847_1023 (size=12104) 2024-11-21T03:24:10,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741849_1025 (size=12001) 2024-11-21T03:24:10,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/7f0bfb8f692148139682554069242fcd 2024-11-21T03:24:10,419 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/c537c84544204758ad3703f4ceb3a914 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c537c84544204758ad3703f4ceb3a914 2024-11-21T03:24:10,425 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/f0fd693f5877443fbf1abaf2792a78ef as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f0fd693f5877443fbf1abaf2792a78ef 2024-11-21T03:24:10,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159510448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159510449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,452 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/A of aebdff1855164d839f0828203eeecd04 into c537c84544204758ad3703f4ceb3a914(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:10,453 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:10,453 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/B of aebdff1855164d839f0828203eeecd04 into f0fd693f5877443fbf1abaf2792a78ef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:10,453 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,456 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/B, priority=13, startTime=1732159450262; duration=0sec 2024-11-21T03:24:10,456 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:10,457 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:B 2024-11-21T03:24:10,457 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/A, priority=13, startTime=1732159450259; duration=0sec 2024-11-21T03:24:10,457 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:10,457 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:10,457 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:A 2024-11-21T03:24:10,461 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:10,461 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/C is initiating minor compaction (all files) 2024-11-21T03:24:10,461 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/C in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,461 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/6ea237037f32450dafa22193dc02857b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a2011bd035204fa2bb860397e441284d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/137a6a94db554fc19d5ba0ace55b5e90] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=35.2 K 2024-11-21T03:24:10,464 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ea237037f32450dafa22193dc02857b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732159447333 2024-11-21T03:24:10,466 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting a2011bd035204fa2bb860397e441284d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732159447634 2024-11-21T03:24:10,467 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 137a6a94db554fc19d5ba0ace55b5e90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732159448849 2024-11-21T03:24:10,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159510449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159510458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/0c493afefcfc41c490b7209a67b53f8a is 50, key is test_row_0/C:col10/1732159450276/Put/seqid=0 2024-11-21T03:24:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159510459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:24:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:10,528 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#C#compaction#14 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:10,529 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/47a092a6d79c4b8984645c521d9b67b7 is 50, key is test_row_0/C:col10/1732159448855/Put/seqid=0 2024-11-21T03:24:10,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741850_1026 (size=12001) 2024-11-21T03:24:10,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/0c493afefcfc41c490b7209a67b53f8a 2024-11-21T03:24:10,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/2df9596da1f3451b80cbe0af83aa4ca4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/2df9596da1f3451b80cbe0af83aa4ca4 2024-11-21T03:24:10,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/2df9596da1f3451b80cbe0af83aa4ca4, entries=150, sequenceid=76, filesize=11.7 K 2024-11-21T03:24:10,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/7f0bfb8f692148139682554069242fcd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7f0bfb8f692148139682554069242fcd 2024-11-21T03:24:10,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741851_1027 (size=12104) 2024-11-21T03:24:10,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7f0bfb8f692148139682554069242fcd, entries=150, sequenceid=76, filesize=11.7 K 2024-11-21T03:24:10,622 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/47a092a6d79c4b8984645c521d9b67b7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/47a092a6d79c4b8984645c521d9b67b7 2024-11-21T03:24:10,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/0c493afefcfc41c490b7209a67b53f8a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0c493afefcfc41c490b7209a67b53f8a 2024-11-21T03:24:10,644 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/C of aebdff1855164d839f0828203eeecd04 into 47a092a6d79c4b8984645c521d9b67b7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:10,645 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:10,645 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/C, priority=13, startTime=1732159450263; duration=0sec 2024-11-21T03:24:10,645 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:10,645 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:C 2024-11-21T03:24:10,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0c493afefcfc41c490b7209a67b53f8a, entries=150, sequenceid=76, filesize=11.7 K 2024-11-21T03:24:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:24:10,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:24:10,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for aebdff1855164d839f0828203eeecd04 in 381ms, sequenceid=76, compaction requested=false 2024-11-21T03:24:10,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:10,662 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-21T03:24:10,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/ddfcb7dafa994cdf9848d4854d358445 is 50, key is test_row_0/A:col10/1732159450313/Put/seqid=0 2024-11-21T03:24:10,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:10,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741852_1028 (size=9657) 2024-11-21T03:24:10,701 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/ddfcb7dafa994cdf9848d4854d358445 2024-11-21T03:24:10,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/6b3ae2158ad24940884da6dbf8faf218 is 50, key is test_row_0/B:col10/1732159450313/Put/seqid=0 2024-11-21T03:24:10,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741853_1029 (size=9657) 2024-11-21T03:24:10,765 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/6b3ae2158ad24940884da6dbf8faf218 2024-11-21T03:24:10,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/41c21515cadf4fffa126cc9ba478bba2 is 50, key is test_row_0/C:col10/1732159450313/Put/seqid=0 2024-11-21T03:24:10,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159510794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159510795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159510798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159510801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159510801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741854_1030 (size=9657) 2024-11-21T03:24:10,820 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/41c21515cadf4fffa126cc9ba478bba2 2024-11-21T03:24:10,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/ddfcb7dafa994cdf9848d4854d358445 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/ddfcb7dafa994cdf9848d4854d358445 2024-11-21T03:24:10,847 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/ddfcb7dafa994cdf9848d4854d358445, entries=100, sequenceid=88, filesize=9.4 K 2024-11-21T03:24:10,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/6b3ae2158ad24940884da6dbf8faf218 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/6b3ae2158ad24940884da6dbf8faf218 2024-11-21T03:24:10,870 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/6b3ae2158ad24940884da6dbf8faf218, entries=100, sequenceid=88, filesize=9.4 K 2024-11-21T03:24:10,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/41c21515cadf4fffa126cc9ba478bba2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/41c21515cadf4fffa126cc9ba478bba2 2024-11-21T03:24:10,895 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/41c21515cadf4fffa126cc9ba478bba2, entries=100, sequenceid=88, filesize=9.4 K 2024-11-21T03:24:10,898 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for aebdff1855164d839f0828203eeecd04 in 235ms, sequenceid=88, compaction requested=true 2024-11-21T03:24:10,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:10,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:10,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-21T03:24:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-21T03:24:10,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-21T03:24:10,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3810 sec 2024-11-21T03:24:10,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3970 sec 2024-11-21T03:24:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:10,931 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:10,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-21T03:24:10,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/f78abce5aadc4da0b125c4620f2e7faa is 50, key is test_row_0/A:col10/1732159450792/Put/seqid=0 2024-11-21T03:24:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159510935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159510936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159510937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159510948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159510958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:10,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741855_1031 (size=16681) 2024-11-21T03:24:10,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/f78abce5aadc4da0b125c4620f2e7faa 2024-11-21T03:24:11,017 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T03:24:11,017 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T03:24:11,020 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T03:24:11,020 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T03:24:11,020 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T03:24:11,020 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T03:24:11,022 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T03:24:11,022 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T03:24:11,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/8aa299cd56414bd5810e42fdaa17307d is 50, key is test_row_0/B:col10/1732159450792/Put/seqid=0 2024-11-21T03:24:11,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741856_1032 (size=12001) 2024-11-21T03:24:11,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/8aa299cd56414bd5810e42fdaa17307d 2024-11-21T03:24:11,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159511068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159511068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159511068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159511088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159511088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/3f06283c2c6f4d3ba010c674772f9152 is 50, key is test_row_0/C:col10/1732159450792/Put/seqid=0 2024-11-21T03:24:11,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741857_1033 (size=12001) 2024-11-21T03:24:11,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/3f06283c2c6f4d3ba010c674772f9152 2024-11-21T03:24:11,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/f78abce5aadc4da0b125c4620f2e7faa as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/f78abce5aadc4da0b125c4620f2e7faa 2024-11-21T03:24:11,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/f78abce5aadc4da0b125c4620f2e7faa, entries=250, sequenceid=117, filesize=16.3 K 2024-11-21T03:24:11,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/8aa299cd56414bd5810e42fdaa17307d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/8aa299cd56414bd5810e42fdaa17307d 2024-11-21T03:24:11,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/8aa299cd56414bd5810e42fdaa17307d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T03:24:11,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/3f06283c2c6f4d3ba010c674772f9152 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3f06283c2c6f4d3ba010c674772f9152 2024-11-21T03:24:11,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3f06283c2c6f4d3ba010c674772f9152, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T03:24:11,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for aebdff1855164d839f0828203eeecd04 in 280ms, sequenceid=117, compaction requested=true 2024-11-21T03:24:11,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:11,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:11,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:11,212 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:11,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:11,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:11,212 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:11,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:11,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:11,218 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50443 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:11,218 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/A is initiating minor compaction (all files) 2024-11-21T03:24:11,218 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/A in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:11,219 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c537c84544204758ad3703f4ceb3a914, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/2df9596da1f3451b80cbe0af83aa4ca4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/ddfcb7dafa994cdf9848d4854d358445, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/f78abce5aadc4da0b125c4620f2e7faa] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=49.3 K 2024-11-21T03:24:11,220 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c537c84544204758ad3703f4ceb3a914, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732159448849 2024-11-21T03:24:11,221 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:11,221 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/B is initiating minor compaction (all files) 2024-11-21T03:24:11,221 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/B in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:11,222 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f0fd693f5877443fbf1abaf2792a78ef, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7f0bfb8f692148139682554069242fcd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/6b3ae2158ad24940884da6dbf8faf218, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/8aa299cd56414bd5810e42fdaa17307d] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=44.7 K 2024-11-21T03:24:11,224 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting f0fd693f5877443fbf1abaf2792a78ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732159448849 2024-11-21T03:24:11,225 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2df9596da1f3451b80cbe0af83aa4ca4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732159450271 2024-11-21T03:24:11,226 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddfcb7dafa994cdf9848d4854d358445, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732159450313 2024-11-21T03:24:11,226 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f0bfb8f692148139682554069242fcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732159450271 2024-11-21T03:24:11,227 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b3ae2158ad24940884da6dbf8faf218, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732159450313 2024-11-21T03:24:11,228 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f78abce5aadc4da0b125c4620f2e7faa, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732159450792 2024-11-21T03:24:11,229 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa299cd56414bd5810e42fdaa17307d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732159450792 2024-11-21T03:24:11,280 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:11,281 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#A#compaction#21 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:11,282 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/3053ef400e784c23a6f68a9e3e6be62f is 50, key is test_row_0/B:col10/1732159450792/Put/seqid=0 2024-11-21T03:24:11,282 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/6f815b1f9b544f2e99eb5f40a18f5077 is 50, key is test_row_0/A:col10/1732159450792/Put/seqid=0 2024-11-21T03:24:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:11,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:11,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:24:11,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741859_1035 (size=12241) 2024-11-21T03:24:11,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741858_1034 (size=12241) 2024-11-21T03:24:11,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/972488cc9928452aa44da13de0228bb3 is 50, key is test_row_0/A:col10/1732159451305/Put/seqid=0 2024-11-21T03:24:11,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159511368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159511368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159511372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741860_1036 (size=12101) 2024-11-21T03:24:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159511375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159511377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/972488cc9928452aa44da13de0228bb3 2024-11-21T03:24:11,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/d0f3b974f72741628c7a6442b207e334 is 50, key is test_row_0/B:col10/1732159451305/Put/seqid=0 2024-11-21T03:24:11,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741861_1037 (size=12101) 2024-11-21T03:24:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159511498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159511499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159511499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159511500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159511501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:24:11,668 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:11,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-21T03:24:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:24:11,675 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:11,677 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:11,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:11,722 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/6f815b1f9b544f2e99eb5f40a18f5077 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6f815b1f9b544f2e99eb5f40a18f5077 2024-11-21T03:24:11,729 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/3053ef400e784c23a6f68a9e3e6be62f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3053ef400e784c23a6f68a9e3e6be62f 2024-11-21T03:24:11,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159511719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159511720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159511720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159511720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159511728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:11,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:11,744 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aebdff1855164d839f0828203eeecd04/A of aebdff1855164d839f0828203eeecd04 into 6f815b1f9b544f2e99eb5f40a18f5077(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:11,744 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:11,744 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/A, priority=12, startTime=1732159451212; duration=0sec 2024-11-21T03:24:11,744 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:11,745 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:A 2024-11-21T03:24:11,745 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:11,749 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aebdff1855164d839f0828203eeecd04/B of aebdff1855164d839f0828203eeecd04 into 3053ef400e784c23a6f68a9e3e6be62f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:11,749 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:11,750 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/B, priority=12, startTime=1732159451212; duration=0sec 2024-11-21T03:24:11,751 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:11,751 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:B 2024-11-21T03:24:11,753 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:11,753 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/C is initiating minor compaction (all files) 2024-11-21T03:24:11,753 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/C in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:11,753 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/47a092a6d79c4b8984645c521d9b67b7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0c493afefcfc41c490b7209a67b53f8a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/41c21515cadf4fffa126cc9ba478bba2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3f06283c2c6f4d3ba010c674772f9152] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=44.7 K 2024-11-21T03:24:11,758 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47a092a6d79c4b8984645c521d9b67b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732159448849 2024-11-21T03:24:11,759 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c493afefcfc41c490b7209a67b53f8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732159450271 2024-11-21T03:24:11,760 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c21515cadf4fffa126cc9ba478bba2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732159450313 2024-11-21T03:24:11,761 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f06283c2c6f4d3ba010c674772f9152, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732159450792 2024-11-21T03:24:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:24:11,807 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#C#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:11,808 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/835971d78103471c850c62a9b62682aa is 50, key is test_row_0/C:col10/1732159450792/Put/seqid=0 2024-11-21T03:24:11,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:24:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:11,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:11,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741862_1038 (size=12241) 2024-11-21T03:24:11,866 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/835971d78103471c850c62a9b62682aa as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/835971d78103471c850c62a9b62682aa 2024-11-21T03:24:11,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/d0f3b974f72741628c7a6442b207e334 2024-11-21T03:24:11,889 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aebdff1855164d839f0828203eeecd04/C of aebdff1855164d839f0828203eeecd04 into 835971d78103471c850c62a9b62682aa(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:11,889 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:11,889 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/C, priority=12, startTime=1732159451212; duration=0sec 2024-11-21T03:24:11,889 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:11,890 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:C 2024-11-21T03:24:11,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/3fd65325858a404e856e2bcc3238a1eb is 50, key is test_row_0/C:col10/1732159451305/Put/seqid=0 2024-11-21T03:24:11,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741863_1039 (size=12101) 2024-11-21T03:24:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:24:11,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:24:11,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:11,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:11,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:12,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159512038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159512047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159512048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159512048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159512048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:24:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:12,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:24:12,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:24:12,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:12,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:12,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:12,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:12,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:12,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/3fd65325858a404e856e2bcc3238a1eb 2024-11-21T03:24:12,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/972488cc9928452aa44da13de0228bb3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/972488cc9928452aa44da13de0228bb3 2024-11-21T03:24:12,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/972488cc9928452aa44da13de0228bb3, entries=150, sequenceid=130, filesize=11.8 K 2024-11-21T03:24:12,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/d0f3b974f72741628c7a6442b207e334 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d0f3b974f72741628c7a6442b207e334 2024-11-21T03:24:12,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d0f3b974f72741628c7a6442b207e334, entries=150, sequenceid=130, filesize=11.8 K 2024-11-21T03:24:12,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/3fd65325858a404e856e2bcc3238a1eb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3fd65325858a404e856e2bcc3238a1eb 2024-11-21T03:24:12,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3fd65325858a404e856e2bcc3238a1eb, entries=150, sequenceid=130, filesize=11.8 K 2024-11-21T03:24:12,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for aebdff1855164d839f0828203eeecd04 in 1130ms, sequenceid=130, compaction requested=false 2024-11-21T03:24:12,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:12,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:24:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:12,477 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:24:12,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/a53394f138a54eb28b00a2203d250231 is 50, key is test_row_0/A:col10/1732159451371/Put/seqid=0 2024-11-21T03:24:12,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741864_1040 (size=12151) 2024-11-21T03:24:12,560 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/a53394f138a54eb28b00a2203d250231 2024-11-21T03:24:12,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:12,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159512610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159512661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159512661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159512663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/a0e453dceb1d4bde80ae403b0686b4eb is 50, key is test_row_0/B:col10/1732159451371/Put/seqid=0 2024-11-21T03:24:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159512668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741865_1041 (size=12151) 2024-11-21T03:24:12,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159512779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159512779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159512779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159512778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159512788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:24:13,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159512998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159512998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159512998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159512999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159512999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,099 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/a0e453dceb1d4bde80ae403b0686b4eb 2024-11-21T03:24:13,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/09d1751cef1e470d92580a16694e6ac8 is 50, key is test_row_0/C:col10/1732159451371/Put/seqid=0 2024-11-21T03:24:13,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741866_1042 (size=12151) 2024-11-21T03:24:13,163 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/09d1751cef1e470d92580a16694e6ac8 2024-11-21T03:24:13,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/a53394f138a54eb28b00a2203d250231 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/a53394f138a54eb28b00a2203d250231 2024-11-21T03:24:13,199 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/a53394f138a54eb28b00a2203d250231, entries=150, sequenceid=156, filesize=11.9 K 2024-11-21T03:24:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/a0e453dceb1d4bde80ae403b0686b4eb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/a0e453dceb1d4bde80ae403b0686b4eb 2024-11-21T03:24:13,212 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/a0e453dceb1d4bde80ae403b0686b4eb, entries=150, sequenceid=156, filesize=11.9 K 2024-11-21T03:24:13,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/09d1751cef1e470d92580a16694e6ac8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/09d1751cef1e470d92580a16694e6ac8 2024-11-21T03:24:13,238 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/09d1751cef1e470d92580a16694e6ac8, entries=150, sequenceid=156, filesize=11.9 K 2024-11-21T03:24:13,241 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for aebdff1855164d839f0828203eeecd04 in 764ms, sequenceid=156, compaction requested=true 2024-11-21T03:24:13,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:13,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:13,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-21T03:24:13,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-21T03:24:13,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-21T03:24:13,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-11-21T03:24:13,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5810 sec 2024-11-21T03:24:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:13,318 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:13,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:24:13,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/29d6721902384e29b0ba794f6feb012f is 50, key is test_row_0/A:col10/1732159453316/Put/seqid=0 2024-11-21T03:24:13,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741867_1043 (size=16931) 2024-11-21T03:24:13,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/29d6721902384e29b0ba794f6feb012f 2024-11-21T03:24:13,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159513382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/d7f4d8ca788e4eb6a3ac1e51bfa2024f is 50, key is test_row_0/B:col10/1732159453316/Put/seqid=0 2024-11-21T03:24:13,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159513393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159513395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159513397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159513398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741868_1044 (size=12151) 2024-11-21T03:24:13,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/d7f4d8ca788e4eb6a3ac1e51bfa2024f 2024-11-21T03:24:13,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/7df0ddbd1f1e4dd4b6dd7b9d1631a4f0 is 50, key is test_row_0/C:col10/1732159453316/Put/seqid=0 2024-11-21T03:24:13,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741869_1045 (size=12151) 2024-11-21T03:24:13,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/7df0ddbd1f1e4dd4b6dd7b9d1631a4f0 2024-11-21T03:24:13,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159513507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159513518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159513518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159513528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159513528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/29d6721902384e29b0ba794f6feb012f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/29d6721902384e29b0ba794f6feb012f 2024-11-21T03:24:13,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/29d6721902384e29b0ba794f6feb012f, entries=250, sequenceid=172, filesize=16.5 K 2024-11-21T03:24:13,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/d7f4d8ca788e4eb6a3ac1e51bfa2024f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d7f4d8ca788e4eb6a3ac1e51bfa2024f 2024-11-21T03:24:13,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d7f4d8ca788e4eb6a3ac1e51bfa2024f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-21T03:24:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/7df0ddbd1f1e4dd4b6dd7b9d1631a4f0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7df0ddbd1f1e4dd4b6dd7b9d1631a4f0 2024-11-21T03:24:13,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7df0ddbd1f1e4dd4b6dd7b9d1631a4f0, entries=150, sequenceid=172, filesize=11.9 K 2024-11-21T03:24:13,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for aebdff1855164d839f0828203eeecd04 in 265ms, sequenceid=172, compaction requested=true 2024-11-21T03:24:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:13,583 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:13,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:13,584 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:13,586 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53424 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:13,586 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/A is initiating minor compaction (all files) 2024-11-21T03:24:13,586 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/A in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:13,587 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6f815b1f9b544f2e99eb5f40a18f5077, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/972488cc9928452aa44da13de0228bb3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/a53394f138a54eb28b00a2203d250231, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/29d6721902384e29b0ba794f6feb012f] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=52.2 K 2024-11-21T03:24:13,587 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:13,587 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/B is initiating minor compaction (all files) 2024-11-21T03:24:13,587 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/B in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:13,587 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3053ef400e784c23a6f68a9e3e6be62f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d0f3b974f72741628c7a6442b207e334, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/a0e453dceb1d4bde80ae403b0686b4eb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d7f4d8ca788e4eb6a3ac1e51bfa2024f] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=47.5 K 2024-11-21T03:24:13,588 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f815b1f9b544f2e99eb5f40a18f5077, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732159450792 2024-11-21T03:24:13,589 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3053ef400e784c23a6f68a9e3e6be62f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732159450792 2024-11-21T03:24:13,589 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 972488cc9928452aa44da13de0228bb3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732159450936 2024-11-21T03:24:13,589 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d0f3b974f72741628c7a6442b207e334, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732159450936 2024-11-21T03:24:13,590 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a53394f138a54eb28b00a2203d250231, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732159451361 2024-11-21T03:24:13,591 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting a0e453dceb1d4bde80ae403b0686b4eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732159451361 2024-11-21T03:24:13,592 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d6721902384e29b0ba794f6feb012f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732159452659 2024-11-21T03:24:13,595 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f4d8ca788e4eb6a3ac1e51bfa2024f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732159452661 2024-11-21T03:24:13,621 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#B#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:13,622 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/ccbea09c0a234aed866db64ca1d315cd is 50, key is test_row_0/B:col10/1732159453316/Put/seqid=0 2024-11-21T03:24:13,628 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#A#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:13,629 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/8720cb8ee47b44e4a9e85c3764eaf829 is 50, key is test_row_0/A:col10/1732159453316/Put/seqid=0 2024-11-21T03:24:13,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741870_1046 (size=12527) 2024-11-21T03:24:13,688 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/ccbea09c0a234aed866db64ca1d315cd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/ccbea09c0a234aed866db64ca1d315cd 2024-11-21T03:24:13,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741871_1047 (size=12527) 2024-11-21T03:24:13,704 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aebdff1855164d839f0828203eeecd04/B of aebdff1855164d839f0828203eeecd04 into ccbea09c0a234aed866db64ca1d315cd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:13,704 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:13,705 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/B, priority=12, startTime=1732159453584; duration=0sec 2024-11-21T03:24:13,705 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:13,705 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:B 2024-11-21T03:24:13,707 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:13,711 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:13,711 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/C is initiating minor compaction (all files) 2024-11-21T03:24:13,712 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/C in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:13,712 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/835971d78103471c850c62a9b62682aa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3fd65325858a404e856e2bcc3238a1eb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/09d1751cef1e470d92580a16694e6ac8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7df0ddbd1f1e4dd4b6dd7b9d1631a4f0] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=47.5 K 2024-11-21T03:24:13,713 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 835971d78103471c850c62a9b62682aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732159450792 2024-11-21T03:24:13,716 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd65325858a404e856e2bcc3238a1eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732159450936 2024-11-21T03:24:13,721 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/8720cb8ee47b44e4a9e85c3764eaf829 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/8720cb8ee47b44e4a9e85c3764eaf829 2024-11-21T03:24:13,721 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d1751cef1e470d92580a16694e6ac8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732159451361 2024-11-21T03:24:13,723 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df0ddbd1f1e4dd4b6dd7b9d1631a4f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732159452661 2024-11-21T03:24:13,737 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aebdff1855164d839f0828203eeecd04/A of aebdff1855164d839f0828203eeecd04 into 8720cb8ee47b44e4a9e85c3764eaf829(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:13,737 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:13,737 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/A, priority=12, startTime=1732159453583; duration=0sec 2024-11-21T03:24:13,737 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:13,737 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:A 2024-11-21T03:24:13,738 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:13,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T03:24:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:13,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/6dc76ed0f02c4f05abbde0db265cd57b is 50, key is test_row_0/A:col10/1732159453392/Put/seqid=0 2024-11-21T03:24:13,763 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:13,764 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/aa028d7449bd4612bde77a1b1a403b38 is 50, key is test_row_0/C:col10/1732159453316/Put/seqid=0 2024-11-21T03:24:13,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159513779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159513781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159513785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159513788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159513793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:24:13,819 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:13,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:13,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741872_1048 (size=14541) 2024-11-21T03:24:13,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-21T03:24:13,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/6dc76ed0f02c4f05abbde0db265cd57b 2024-11-21T03:24:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T03:24:13,827 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:13,829 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:13,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:13,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741873_1049 (size=12527) 2024-11-21T03:24:13,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/f1a8d6b2cb4d4ed5a99438bafe27f392 is 50, key is test_row_0/B:col10/1732159453392/Put/seqid=0 2024-11-21T03:24:13,857 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/aa028d7449bd4612bde77a1b1a403b38 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/aa028d7449bd4612bde77a1b1a403b38 2024-11-21T03:24:13,869 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aebdff1855164d839f0828203eeecd04/C of aebdff1855164d839f0828203eeecd04 into aa028d7449bd4612bde77a1b1a403b38(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:13,869 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:13,869 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/C, priority=12, startTime=1732159453584; duration=0sec 2024-11-21T03:24:13,870 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:13,870 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:C 2024-11-21T03:24:13,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159513898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741874_1050 (size=12151) 2024-11-21T03:24:13,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/f1a8d6b2cb4d4ed5a99438bafe27f392 2024-11-21T03:24:13,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159513908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159513908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159513909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159513908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T03:24:13,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/c49c5f321e8d480881a28694d564b673 is 50, key is test_row_0/C:col10/1732159453392/Put/seqid=0 2024-11-21T03:24:13,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T03:24:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:13,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:13,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:13,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:13,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741875_1051 (size=12151) 2024-11-21T03:24:13,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/c49c5f321e8d480881a28694d564b673 2024-11-21T03:24:14,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/6dc76ed0f02c4f05abbde0db265cd57b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6dc76ed0f02c4f05abbde0db265cd57b 2024-11-21T03:24:14,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6dc76ed0f02c4f05abbde0db265cd57b, entries=200, sequenceid=196, filesize=14.2 K 2024-11-21T03:24:14,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/f1a8d6b2cb4d4ed5a99438bafe27f392 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f1a8d6b2cb4d4ed5a99438bafe27f392 2024-11-21T03:24:14,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f1a8d6b2cb4d4ed5a99438bafe27f392, entries=150, sequenceid=196, filesize=11.9 K 2024-11-21T03:24:14,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/c49c5f321e8d480881a28694d564b673 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/c49c5f321e8d480881a28694d564b673 2024-11-21T03:24:14,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/c49c5f321e8d480881a28694d564b673, entries=150, sequenceid=196, filesize=11.9 K 2024-11-21T03:24:14,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for aebdff1855164d839f0828203eeecd04 in 315ms, sequenceid=196, compaction requested=false 2024-11-21T03:24:14,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:14,122 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:14,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T03:24:14,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/abfc7e0f43334b9bb8777e53884b5ba7 is 50, key is test_row_0/A:col10/1732159453784/Put/seqid=0 2024-11-21T03:24:14,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T03:24:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T03:24:14,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741876_1052 (size=14541) 2024-11-21T03:24:14,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159514172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159514172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159514174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159514177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159514179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/abfc7e0f43334b9bb8777e53884b5ba7 2024-11-21T03:24:14,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/42136dddf2aa47bdb1567dab3d07fd5c is 50, key is test_row_0/B:col10/1732159453784/Put/seqid=0 2024-11-21T03:24:14,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741877_1053 (size=12151) 2024-11-21T03:24:14,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159514288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159514288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T03:24:14,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159514288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159514289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159514288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T03:24:14,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:14,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T03:24:14,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159514498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159514497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159514508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159514509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159514518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T03:24:14,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:14,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/42136dddf2aa47bdb1567dab3d07fd5c 2024-11-21T03:24:14,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/909a83b8189f496cb8279f612835c6c8 is 50, key is test_row_0/C:col10/1732159453784/Put/seqid=0 2024-11-21T03:24:14,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T03:24:14,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:14,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741878_1054 (size=12151) 2024-11-21T03:24:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:14,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/909a83b8189f496cb8279f612835c6c8 2024-11-21T03:24:14,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/abfc7e0f43334b9bb8777e53884b5ba7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/abfc7e0f43334b9bb8777e53884b5ba7 2024-11-21T03:24:14,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159514819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/abfc7e0f43334b9bb8777e53884b5ba7, entries=200, sequenceid=213, filesize=14.2 K 2024-11-21T03:24:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159514819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/42136dddf2aa47bdb1567dab3d07fd5c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/42136dddf2aa47bdb1567dab3d07fd5c 2024-11-21T03:24:14,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159514819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159514819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159514828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:14,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/42136dddf2aa47bdb1567dab3d07fd5c, entries=150, sequenceid=213, filesize=11.9 K 2024-11-21T03:24:14,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/909a83b8189f496cb8279f612835c6c8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/909a83b8189f496cb8279f612835c6c8 2024-11-21T03:24:14,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/909a83b8189f496cb8279f612835c6c8, entries=150, sequenceid=213, filesize=11.9 K 2024-11-21T03:24:14,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for aebdff1855164d839f0828203eeecd04 in 744ms, sequenceid=213, compaction requested=true 2024-11-21T03:24:14,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:14,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:14,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:14,867 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:14,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:14,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:14,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:14,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:24:14,868 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:14,880 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:14,880 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/A is initiating minor compaction (all files) 2024-11-21T03:24:14,880 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/A in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,880 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/8720cb8ee47b44e4a9e85c3764eaf829, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6dc76ed0f02c4f05abbde0db265cd57b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/abfc7e0f43334b9bb8777e53884b5ba7] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=40.6 K 2024-11-21T03:24:14,881 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:14,881 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/B is initiating minor compaction (all files) 2024-11-21T03:24:14,881 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/B in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,882 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/ccbea09c0a234aed866db64ca1d315cd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f1a8d6b2cb4d4ed5a99438bafe27f392, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/42136dddf2aa47bdb1567dab3d07fd5c] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.0 K 2024-11-21T03:24:14,882 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8720cb8ee47b44e4a9e85c3764eaf829, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732159452661 2024-11-21T03:24:14,883 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccbea09c0a234aed866db64ca1d315cd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732159452661 2024-11-21T03:24:14,884 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc76ed0f02c4f05abbde0db265cd57b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732159453390 2024-11-21T03:24:14,884 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a8d6b2cb4d4ed5a99438bafe27f392, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732159453392 2024-11-21T03:24:14,885 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting abfc7e0f43334b9bb8777e53884b5ba7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732159453773 2024-11-21T03:24:14,886 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42136dddf2aa47bdb1567dab3d07fd5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732159453784 2024-11-21T03:24:14,923 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:14,924 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/405fa82743ef41659cf05e4e8f8678de is 50, key is test_row_0/A:col10/1732159453784/Put/seqid=0 2024-11-21T03:24:14,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T03:24:14,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:14,926 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T03:24:14,931 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#B#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:14,932 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/28c7a07d049f4310bdbe2ed9badf7cdd is 50, key is test_row_0/B:col10/1732159453784/Put/seqid=0 2024-11-21T03:24:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T03:24:14,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741879_1055 (size=12629) 2024-11-21T03:24:14,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741880_1056 (size=12629) 2024-11-21T03:24:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/cba9df26be5d4ef6abf445953f27c4cd is 50, key is test_row_0/A:col10/1732159454171/Put/seqid=0 2024-11-21T03:24:15,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741881_1057 (size=12151) 2024-11-21T03:24:15,035 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/cba9df26be5d4ef6abf445953f27c4cd 2024-11-21T03:24:15,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/e48c87ec53eb428ba287565a128023f9 is 50, key is test_row_0/B:col10/1732159454171/Put/seqid=0 2024-11-21T03:24:15,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741882_1058 (size=12151) 2024-11-21T03:24:15,127 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/e48c87ec53eb428ba287565a128023f9 2024-11-21T03:24:15,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/ce9841549e634b9fa7e916d4c1c346bc is 50, key is test_row_0/C:col10/1732159454171/Put/seqid=0 2024-11-21T03:24:15,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741883_1059 (size=12151) 2024-11-21T03:24:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:15,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:15,396 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/405fa82743ef41659cf05e4e8f8678de as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/405fa82743ef41659cf05e4e8f8678de 2024-11-21T03:24:15,407 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/28c7a07d049f4310bdbe2ed9badf7cdd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/28c7a07d049f4310bdbe2ed9badf7cdd 2024-11-21T03:24:15,444 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/B of aebdff1855164d839f0828203eeecd04 into 28c7a07d049f4310bdbe2ed9badf7cdd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:15,444 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:15,444 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/B, priority=13, startTime=1732159454867; duration=0sec 2024-11-21T03:24:15,445 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:15,445 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:B 2024-11-21T03:24:15,445 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:15,447 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/A of aebdff1855164d839f0828203eeecd04 into 405fa82743ef41659cf05e4e8f8678de(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:15,447 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:15,447 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/A, priority=13, startTime=1732159454867; duration=0sec 2024-11-21T03:24:15,447 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:15,447 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/C is initiating minor compaction (all files) 2024-11-21T03:24:15,447 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/C in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:15,447 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:15,447 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:A 2024-11-21T03:24:15,447 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/aa028d7449bd4612bde77a1b1a403b38, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/c49c5f321e8d480881a28694d564b673, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/909a83b8189f496cb8279f612835c6c8] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.0 K 2024-11-21T03:24:15,448 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa028d7449bd4612bde77a1b1a403b38, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732159452661 2024-11-21T03:24:15,449 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49c5f321e8d480881a28694d564b673, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732159453392 2024-11-21T03:24:15,450 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 909a83b8189f496cb8279f612835c6c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732159453784 2024-11-21T03:24:15,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159515448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159515456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,467 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#C#compaction#47 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:15,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159515460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,467 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/8c9bf3edc560403ca9c90d6a359e823f is 50, key is test_row_0/C:col10/1732159453784/Put/seqid=0 2024-11-21T03:24:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159515466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159515473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741884_1060 (size=12629) 2024-11-21T03:24:15,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159515579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159515579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159515580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159515579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159515591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,608 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/ce9841549e634b9fa7e916d4c1c346bc 2024-11-21T03:24:15,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/cba9df26be5d4ef6abf445953f27c4cd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cba9df26be5d4ef6abf445953f27c4cd 2024-11-21T03:24:15,635 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cba9df26be5d4ef6abf445953f27c4cd, entries=150, sequenceid=233, filesize=11.9 K 2024-11-21T03:24:15,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/e48c87ec53eb428ba287565a128023f9 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/e48c87ec53eb428ba287565a128023f9 2024-11-21T03:24:15,657 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/e48c87ec53eb428ba287565a128023f9, entries=150, sequenceid=233, filesize=11.9 K 2024-11-21T03:24:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/ce9841549e634b9fa7e916d4c1c346bc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/ce9841549e634b9fa7e916d4c1c346bc 2024-11-21T03:24:15,689 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/ce9841549e634b9fa7e916d4c1c346bc, entries=150, sequenceid=233, filesize=11.9 K 2024-11-21T03:24:15,693 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for aebdff1855164d839f0828203eeecd04 in 767ms, sequenceid=233, compaction requested=false 2024-11-21T03:24:15,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:15,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:15,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-21T03:24:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-21T03:24:15,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T03:24:15,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-21T03:24:15,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.8800 sec 2024-11-21T03:24:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:15,805 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:15,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T03:24:15,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/20d03db84d9f45fda09f1fdb8cf99f88 is 50, key is test_row_0/A:col10/1732159455462/Put/seqid=0 2024-11-21T03:24:15,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159515857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159515860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159515863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741885_1061 (size=12151) 2024-11-21T03:24:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159515867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159515871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,913 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/8c9bf3edc560403ca9c90d6a359e823f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8c9bf3edc560403ca9c90d6a359e823f 2024-11-21T03:24:15,923 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/C of aebdff1855164d839f0828203eeecd04 into 8c9bf3edc560403ca9c90d6a359e823f(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:15,923 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:15,923 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/C, priority=13, startTime=1732159454867; duration=0sec 2024-11-21T03:24:15,923 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:15,923 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:C 2024-11-21T03:24:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T03:24:15,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,980 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159515978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159515978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:15,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-21T03:24:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:24:15,990 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:15,991 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:15,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159515988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159515988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159515996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:24:16,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:24:16,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:16,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159516189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159516189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159516208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159516208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159516218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/20d03db84d9f45fda09f1fdb8cf99f88 2024-11-21T03:24:16,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/54510f14816e4e52b87cb13ea118dc4e is 50, key is test_row_0/B:col10/1732159455462/Put/seqid=0 2024-11-21T03:24:16,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741886_1062 (size=12151) 2024-11-21T03:24:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:24:16,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:24:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:24:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159516507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159516507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159516521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159516529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159516529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:24:16,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:24:16,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:16,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/54510f14816e4e52b87cb13ea118dc4e 2024-11-21T03:24:16,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/7eac5f4bd0ba4b5bafbbfde5ac52ae5c is 50, key is test_row_0/C:col10/1732159455462/Put/seqid=0 2024-11-21T03:24:16,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:24:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741887_1063 (size=12151) 2024-11-21T03:24:16,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:24:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:16,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:16,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:16,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:17,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159517019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:17,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:17,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:17,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:17,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159517020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:17,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:17,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159517037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:17,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159517037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:17,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159517048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:17,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:24:17,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:17,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:17,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:17,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:17,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:24:17,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/7eac5f4bd0ba4b5bafbbfde5ac52ae5c 2024-11-21T03:24:17,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/20d03db84d9f45fda09f1fdb8cf99f88 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/20d03db84d9f45fda09f1fdb8cf99f88 2024-11-21T03:24:17,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/20d03db84d9f45fda09f1fdb8cf99f88, entries=150, sequenceid=254, filesize=11.9 K 2024-11-21T03:24:17,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/54510f14816e4e52b87cb13ea118dc4e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/54510f14816e4e52b87cb13ea118dc4e 2024-11-21T03:24:17,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:24:17,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:17,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:17,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:17,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:17,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:17,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/54510f14816e4e52b87cb13ea118dc4e, entries=150, sequenceid=254, filesize=11.9 K 2024-11-21T03:24:17,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/7eac5f4bd0ba4b5bafbbfde5ac52ae5c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7eac5f4bd0ba4b5bafbbfde5ac52ae5c 2024-11-21T03:24:17,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7eac5f4bd0ba4b5bafbbfde5ac52ae5c, entries=150, sequenceid=254, filesize=11.9 K 2024-11-21T03:24:17,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for aebdff1855164d839f0828203eeecd04 in 1485ms, sequenceid=254, compaction requested=true 2024-11-21T03:24:17,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:17,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:17,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:17,291 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:17,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:17,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:17,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:17,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:24:17,291 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:17,293 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:17,294 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/A is initiating minor compaction (all files) 2024-11-21T03:24:17,294 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/A in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:17,294 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/405fa82743ef41659cf05e4e8f8678de, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cba9df26be5d4ef6abf445953f27c4cd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/20d03db84d9f45fda09f1fdb8cf99f88] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.1 K 2024-11-21T03:24:17,294 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:17,294 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/B is initiating minor compaction (all files) 2024-11-21T03:24:17,294 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/B in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:17,294 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/28c7a07d049f4310bdbe2ed9badf7cdd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/e48c87ec53eb428ba287565a128023f9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/54510f14816e4e52b87cb13ea118dc4e] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.1 K 2024-11-21T03:24:17,295 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 405fa82743ef41659cf05e4e8f8678de, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732159453784 2024-11-21T03:24:17,295 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 28c7a07d049f4310bdbe2ed9badf7cdd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732159453784 2024-11-21T03:24:17,295 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting cba9df26be5d4ef6abf445953f27c4cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732159454165 2024-11-21T03:24:17,295 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e48c87ec53eb428ba287565a128023f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732159454165 2024-11-21T03:24:17,296 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 54510f14816e4e52b87cb13ea118dc4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159455462 2024-11-21T03:24:17,297 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20d03db84d9f45fda09f1fdb8cf99f88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159455462 2024-11-21T03:24:17,330 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:17,332 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:17,335 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/d205f7e484da4eddb4b1d5535430a531 is 50, key is test_row_0/B:col10/1732159455462/Put/seqid=0 2024-11-21T03:24:17,335 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/11985f7157d544e89461ae0b3c57946d is 50, key is test_row_0/A:col10/1732159455462/Put/seqid=0 2024-11-21T03:24:17,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741889_1065 (size=12731) 2024-11-21T03:24:17,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741888_1064 (size=12731) 2024-11-21T03:24:17,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:24:17,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:17,411 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T03:24:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e5cba214774641e3b39732d40af497f1 is 50, key is test_row_0/A:col10/1732159455862/Put/seqid=0 2024-11-21T03:24:17,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741890_1066 (size=12301) 2024-11-21T03:24:17,784 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/11985f7157d544e89461ae0b3c57946d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/11985f7157d544e89461ae0b3c57946d 2024-11-21T03:24:17,796 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/d205f7e484da4eddb4b1d5535430a531 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d205f7e484da4eddb4b1d5535430a531 2024-11-21T03:24:17,805 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/A of aebdff1855164d839f0828203eeecd04 into 11985f7157d544e89461ae0b3c57946d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:17,805 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:17,805 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/A, priority=13, startTime=1732159457291; duration=0sec 2024-11-21T03:24:17,805 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:17,805 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:A 2024-11-21T03:24:17,805 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:17,809 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:17,809 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/C is initiating minor compaction (all files) 2024-11-21T03:24:17,809 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/C in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:17,809 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8c9bf3edc560403ca9c90d6a359e823f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/ce9841549e634b9fa7e916d4c1c346bc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7eac5f4bd0ba4b5bafbbfde5ac52ae5c] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.1 K 2024-11-21T03:24:17,810 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c9bf3edc560403ca9c90d6a359e823f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732159453784 2024-11-21T03:24:17,811 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce9841549e634b9fa7e916d4c1c346bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732159454165 2024-11-21T03:24:17,811 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eac5f4bd0ba4b5bafbbfde5ac52ae5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159455462 2024-11-21T03:24:17,821 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/B of aebdff1855164d839f0828203eeecd04 into d205f7e484da4eddb4b1d5535430a531(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:17,821 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:17,821 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/B, priority=13, startTime=1732159457291; duration=0sec 2024-11-21T03:24:17,821 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:17,821 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:B 2024-11-21T03:24:17,837 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#C#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:17,839 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/48a81594b5aa40d8bd74a60deb6470f2 is 50, key is test_row_0/C:col10/1732159455462/Put/seqid=0 2024-11-21T03:24:17,854 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e5cba214774641e3b39732d40af497f1 2024-11-21T03:24:17,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741891_1067 (size=12731) 2024-11-21T03:24:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/0deb3bf4ef4342eca8f6c052bd9abcfe is 50, key is test_row_0/B:col10/1732159455862/Put/seqid=0 2024-11-21T03:24:17,903 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/48a81594b5aa40d8bd74a60deb6470f2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/48a81594b5aa40d8bd74a60deb6470f2 2024-11-21T03:24:17,913 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/C of aebdff1855164d839f0828203eeecd04 into 48a81594b5aa40d8bd74a60deb6470f2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:17,913 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:17,914 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/C, priority=13, startTime=1732159457291; duration=0sec 2024-11-21T03:24:17,914 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:17,914 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:C 2024-11-21T03:24:17,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741892_1068 (size=12301) 2024-11-21T03:24:18,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:18,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159518058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159518058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159518058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,066 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_1' in aebdff1855164d839f0828203eeecd04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:18,066 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in aebdff1855164d839f0828203eeecd04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159518062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159518067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,069 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_1' in aebdff1855164d839f0828203eeecd04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:24:18,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159518168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159518178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,328 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/0deb3bf4ef4342eca8f6c052bd9abcfe 2024-11-21T03:24:18,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/0355cf2260ef45a8829a028f1f8a3e9f is 50, key is test_row_0/C:col10/1732159455862/Put/seqid=0 2024-11-21T03:24:18,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159518379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159518387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741893_1069 (size=12301) 2024-11-21T03:24:18,417 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/0355cf2260ef45a8829a028f1f8a3e9f 2024-11-21T03:24:18,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e5cba214774641e3b39732d40af497f1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e5cba214774641e3b39732d40af497f1 2024-11-21T03:24:18,460 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e5cba214774641e3b39732d40af497f1, entries=150, sequenceid=273, filesize=12.0 K 2024-11-21T03:24:18,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/0deb3bf4ef4342eca8f6c052bd9abcfe as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0deb3bf4ef4342eca8f6c052bd9abcfe 2024-11-21T03:24:18,474 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0deb3bf4ef4342eca8f6c052bd9abcfe, entries=150, sequenceid=273, filesize=12.0 K 2024-11-21T03:24:18,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/0355cf2260ef45a8829a028f1f8a3e9f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0355cf2260ef45a8829a028f1f8a3e9f 2024-11-21T03:24:18,484 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0355cf2260ef45a8829a028f1f8a3e9f, entries=150, sequenceid=273, filesize=12.0 K 2024-11-21T03:24:18,487 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for aebdff1855164d839f0828203eeecd04 in 1076ms, sequenceid=273, compaction requested=false 2024-11-21T03:24:18,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:18,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:18,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-21T03:24:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-21T03:24:18,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-21T03:24:18,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4990 sec 2024-11-21T03:24:18,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.5170 sec 2024-11-21T03:24:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:18,691 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:18,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T03:24:18,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/41a69cde80954041b50011dceff21e01 is 50, key is test_row_0/A:col10/1732159458060/Put/seqid=0 2024-11-21T03:24:18,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741894_1070 (size=12301) 2024-11-21T03:24:18,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/41a69cde80954041b50011dceff21e01 2024-11-21T03:24:18,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/7b1506433e25449bab53dbac9d715c5f is 50, key is test_row_0/B:col10/1732159458060/Put/seqid=0 2024-11-21T03:24:18,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159518766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159518766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741895_1071 (size=12301) 2024-11-21T03:24:18,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159518878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159518878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:19,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159519099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159519099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:19,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/7b1506433e25449bab53dbac9d715c5f 2024-11-21T03:24:19,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/97bec831e2c14e5a88c30601a467170a is 50, key is test_row_0/C:col10/1732159458060/Put/seqid=0 2024-11-21T03:24:19,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741896_1072 (size=12301) 2024-11-21T03:24:19,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/97bec831e2c14e5a88c30601a467170a 2024-11-21T03:24:19,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/41a69cde80954041b50011dceff21e01 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/41a69cde80954041b50011dceff21e01 2024-11-21T03:24:19,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/41a69cde80954041b50011dceff21e01, entries=150, sequenceid=295, filesize=12.0 K 2024-11-21T03:24:19,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/7b1506433e25449bab53dbac9d715c5f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7b1506433e25449bab53dbac9d715c5f 2024-11-21T03:24:19,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7b1506433e25449bab53dbac9d715c5f, entries=150, sequenceid=295, filesize=12.0 K 2024-11-21T03:24:19,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/97bec831e2c14e5a88c30601a467170a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/97bec831e2c14e5a88c30601a467170a 2024-11-21T03:24:19,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/97bec831e2c14e5a88c30601a467170a, entries=150, sequenceid=295, filesize=12.0 K 2024-11-21T03:24:19,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for aebdff1855164d839f0828203eeecd04 in 579ms, sequenceid=295, compaction requested=true 2024-11-21T03:24:19,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:19,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:19,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:19,271 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:19,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:19,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:19,271 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:19,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:19,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:19,272 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:19,272 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/A is initiating minor compaction (all files) 2024-11-21T03:24:19,272 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/A in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:19,273 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/11985f7157d544e89461ae0b3c57946d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e5cba214774641e3b39732d40af497f1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/41a69cde80954041b50011dceff21e01] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.5 K 2024-11-21T03:24:19,274 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:19,274 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/B is initiating minor compaction (all files) 2024-11-21T03:24:19,274 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/B in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:19,274 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d205f7e484da4eddb4b1d5535430a531, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0deb3bf4ef4342eca8f6c052bd9abcfe, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7b1506433e25449bab53dbac9d715c5f] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.5 K 2024-11-21T03:24:19,275 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11985f7157d544e89461ae0b3c57946d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159455462 2024-11-21T03:24:19,275 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d205f7e484da4eddb4b1d5535430a531, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159455462 2024-11-21T03:24:19,276 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5cba214774641e3b39732d40af497f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732159455853 2024-11-21T03:24:19,276 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0deb3bf4ef4342eca8f6c052bd9abcfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732159455853 2024-11-21T03:24:19,276 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b1506433e25449bab53dbac9d715c5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732159458056 2024-11-21T03:24:19,277 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a69cde80954041b50011dceff21e01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732159458056 2024-11-21T03:24:19,287 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:19,288 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:19,288 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/d6d2e66be4ec4af880da9500cb895855 is 50, key is test_row_0/A:col10/1732159458060/Put/seqid=0 2024-11-21T03:24:19,290 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/fff9c821101045ac9f90552f5aaac805 is 50, key is test_row_0/B:col10/1732159458060/Put/seqid=0 2024-11-21T03:24:19,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741897_1073 (size=12983) 2024-11-21T03:24:19,300 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/d6d2e66be4ec4af880da9500cb895855 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/d6d2e66be4ec4af880da9500cb895855 2024-11-21T03:24:19,309 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/A of aebdff1855164d839f0828203eeecd04 into d6d2e66be4ec4af880da9500cb895855(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:19,309 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:19,309 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/A, priority=13, startTime=1732159459271; duration=0sec 2024-11-21T03:24:19,309 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:19,309 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:A 2024-11-21T03:24:19,309 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:19,310 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:19,311 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/C is initiating minor compaction (all files) 2024-11-21T03:24:19,311 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/C in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:19,311 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/48a81594b5aa40d8bd74a60deb6470f2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0355cf2260ef45a8829a028f1f8a3e9f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/97bec831e2c14e5a88c30601a467170a] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.5 K 2024-11-21T03:24:19,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741898_1074 (size=12983) 2024-11-21T03:24:19,312 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a81594b5aa40d8bd74a60deb6470f2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159455462 2024-11-21T03:24:19,312 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0355cf2260ef45a8829a028f1f8a3e9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732159455853 2024-11-21T03:24:19,313 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97bec831e2c14e5a88c30601a467170a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732159458056 2024-11-21T03:24:19,322 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/fff9c821101045ac9f90552f5aaac805 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/fff9c821101045ac9f90552f5aaac805 2024-11-21T03:24:19,329 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:19,330 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/b800e00fe2344982b793affbdfaa9f3d is 50, key is test_row_0/C:col10/1732159458060/Put/seqid=0 2024-11-21T03:24:19,334 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/B of aebdff1855164d839f0828203eeecd04 into fff9c821101045ac9f90552f5aaac805(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:19,334 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:19,334 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/B, priority=13, startTime=1732159459271; duration=0sec 2024-11-21T03:24:19,335 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:19,335 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:B 2024-11-21T03:24:19,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741899_1075 (size=12983) 2024-11-21T03:24:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:19,410 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:19,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T03:24:19,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/873113d059fd45bda78b806809c4aaa3 is 50, key is test_row_0/A:col10/1732159458749/Put/seqid=0 2024-11-21T03:24:19,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741900_1076 (size=14741) 2024-11-21T03:24:19,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159519459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:19,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159519461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:19,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159519578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:19,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159519578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:19,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:19,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:19,752 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/b800e00fe2344982b793affbdfaa9f3d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/b800e00fe2344982b793affbdfaa9f3d 2024-11-21T03:24:19,760 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/C of aebdff1855164d839f0828203eeecd04 into b800e00fe2344982b793affbdfaa9f3d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:19,760 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:19,760 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/C, priority=13, startTime=1732159459271; duration=0sec 2024-11-21T03:24:19,760 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:19,760 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:C 2024-11-21T03:24:19,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159519788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:19,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159519788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:19,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/873113d059fd45bda78b806809c4aaa3 2024-11-21T03:24:19,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/c25fb1487295489eabac40523bce3443 is 50, key is test_row_0/B:col10/1732159458749/Put/seqid=0 2024-11-21T03:24:19,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741901_1077 (size=12301) 2024-11-21T03:24:20,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159520087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159520088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159520088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:20,090 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_1' in aebdff1855164d839f0828203eeecd04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:20,090 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_1' in aebdff1855164d839f0828203eeecd04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,090 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in aebdff1855164d839f0828203eeecd04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:20,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159520097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159520098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:24:20,169 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:20,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-21T03:24:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:24:20,180 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:20,182 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:20,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:20,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/c25fb1487295489eabac40523bce3443 2024-11-21T03:24:20,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/717fd5735dbc4ae686f15cd6e8279bf4 is 50, key is test_row_0/C:col10/1732159458749/Put/seqid=0 2024-11-21T03:24:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:24:20,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741902_1078 (size=12301) 2024-11-21T03:24:20,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/717fd5735dbc4ae686f15cd6e8279bf4 2024-11-21T03:24:20,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/873113d059fd45bda78b806809c4aaa3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/873113d059fd45bda78b806809c4aaa3 2024-11-21T03:24:20,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/873113d059fd45bda78b806809c4aaa3, entries=200, sequenceid=315, filesize=14.4 K 2024-11-21T03:24:20,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/c25fb1487295489eabac40523bce3443 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/c25fb1487295489eabac40523bce3443 2024-11-21T03:24:20,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/c25fb1487295489eabac40523bce3443, entries=150, sequenceid=315, filesize=12.0 K 2024-11-21T03:24:20,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/717fd5735dbc4ae686f15cd6e8279bf4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/717fd5735dbc4ae686f15cd6e8279bf4 2024-11-21T03:24:20,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/717fd5735dbc4ae686f15cd6e8279bf4, entries=150, sequenceid=315, filesize=12.0 K 2024-11-21T03:24:20,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for aebdff1855164d839f0828203eeecd04 in 904ms, sequenceid=315, compaction requested=false 2024-11-21T03:24:20,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:20,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T03:24:20,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:20,337 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T03:24:20,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/cad09c118a7d4fb4a26247696713c8db is 50, key is test_row_0/A:col10/1732159459451/Put/seqid=0 2024-11-21T03:24:20,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741903_1079 (size=12301) 2024-11-21T03:24:20,361 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/cad09c118a7d4fb4a26247696713c8db 2024-11-21T03:24:20,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/5dabf226a4c544459f5fd40580659b2a is 50, key is test_row_0/B:col10/1732159459451/Put/seqid=0 2024-11-21T03:24:20,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741904_1080 (size=12301) 2024-11-21T03:24:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:24:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:20,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:20,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159520652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159520654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:20,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159520757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159520757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:20,792 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/5dabf226a4c544459f5fd40580659b2a 2024-11-21T03:24:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:24:20,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/72c47b93b915462a90b753b1330ef39e is 50, key is test_row_0/C:col10/1732159459451/Put/seqid=0 2024-11-21T03:24:20,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741905_1081 (size=12301) 2024-11-21T03:24:20,807 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/72c47b93b915462a90b753b1330ef39e 2024-11-21T03:24:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/cad09c118a7d4fb4a26247696713c8db as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cad09c118a7d4fb4a26247696713c8db 2024-11-21T03:24:20,818 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cad09c118a7d4fb4a26247696713c8db, entries=150, sequenceid=335, filesize=12.0 K 2024-11-21T03:24:20,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/5dabf226a4c544459f5fd40580659b2a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/5dabf226a4c544459f5fd40580659b2a 2024-11-21T03:24:20,825 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/5dabf226a4c544459f5fd40580659b2a, entries=150, sequenceid=335, filesize=12.0 K 2024-11-21T03:24:20,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/72c47b93b915462a90b753b1330ef39e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/72c47b93b915462a90b753b1330ef39e 2024-11-21T03:24:20,834 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/72c47b93b915462a90b753b1330ef39e, entries=150, sequenceid=335, filesize=12.0 K 2024-11-21T03:24:20,836 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for aebdff1855164d839f0828203eeecd04 in 499ms, sequenceid=335, compaction requested=true 2024-11-21T03:24:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-21T03:24:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-21T03:24:20,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-21T03:24:20,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 656 msec 2024-11-21T03:24:20,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 672 msec 2024-11-21T03:24:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:20,972 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:20,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T03:24:20,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/c1ceb52ec0d942ff87cf89b983117b4f is 50, key is test_row_0/A:col10/1732159460970/Put/seqid=0 2024-11-21T03:24:21,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741906_1082 (size=12301) 2024-11-21T03:24:21,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/c1ceb52ec0d942ff87cf89b983117b4f 2024-11-21T03:24:21,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159521030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:21,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159521033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:21,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/87d71e6cc0c34579b1176717e9564071 is 50, key is test_row_0/B:col10/1732159460970/Put/seqid=0 2024-11-21T03:24:21,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741907_1083 (size=12301) 2024-11-21T03:24:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159521139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:21,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159521147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:24:21,308 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:21,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-21T03:24:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:24:21,310 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:21,311 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:21,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:21,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159521348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:21,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159521357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:24:21,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T03:24:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:21,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:21,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/87d71e6cc0c34579b1176717e9564071 2024-11-21T03:24:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:21,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/8fb6d181ebba4a6ea88b71b16f431df0 is 50, key is test_row_0/C:col10/1732159460970/Put/seqid=0 2024-11-21T03:24:21,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741908_1084 (size=12301) 2024-11-21T03:24:21,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/8fb6d181ebba4a6ea88b71b16f431df0 2024-11-21T03:24:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/c1ceb52ec0d942ff87cf89b983117b4f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c1ceb52ec0d942ff87cf89b983117b4f 2024-11-21T03:24:21,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c1ceb52ec0d942ff87cf89b983117b4f, entries=150, sequenceid=355, filesize=12.0 K 2024-11-21T03:24:21,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/87d71e6cc0c34579b1176717e9564071 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/87d71e6cc0c34579b1176717e9564071 2024-11-21T03:24:21,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/87d71e6cc0c34579b1176717e9564071, entries=150, sequenceid=355, filesize=12.0 K 2024-11-21T03:24:21,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/8fb6d181ebba4a6ea88b71b16f431df0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8fb6d181ebba4a6ea88b71b16f431df0 2024-11-21T03:24:21,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8fb6d181ebba4a6ea88b71b16f431df0, entries=150, sequenceid=355, filesize=12.0 K 2024-11-21T03:24:21,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for aebdff1855164d839f0828203eeecd04 in 551ms, sequenceid=355, compaction requested=true 2024-11-21T03:24:21,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:21,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:21,523 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:21,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:21,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:21,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:21,524 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:21,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:21,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:21,525 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:21,525 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:21,525 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/B is initiating minor compaction (all files) 2024-11-21T03:24:21,525 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/A is initiating minor compaction (all files) 2024-11-21T03:24:21,525 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/A in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:21,525 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/B in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:21,526 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/d6d2e66be4ec4af880da9500cb895855, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/873113d059fd45bda78b806809c4aaa3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cad09c118a7d4fb4a26247696713c8db, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c1ceb52ec0d942ff87cf89b983117b4f] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=51.1 K 2024-11-21T03:24:21,526 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/fff9c821101045ac9f90552f5aaac805, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/c25fb1487295489eabac40523bce3443, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/5dabf226a4c544459f5fd40580659b2a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/87d71e6cc0c34579b1176717e9564071] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=48.7 K 2024-11-21T03:24:21,526 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting fff9c821101045ac9f90552f5aaac805, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732159458056 2024-11-21T03:24:21,526 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6d2e66be4ec4af880da9500cb895855, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732159458056 2024-11-21T03:24:21,526 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting c25fb1487295489eabac40523bce3443, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732159458749 2024-11-21T03:24:21,526 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 873113d059fd45bda78b806809c4aaa3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732159458749 2024-11-21T03:24:21,527 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dabf226a4c544459f5fd40580659b2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732159459451 2024-11-21T03:24:21,527 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting cad09c118a7d4fb4a26247696713c8db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732159459451 2024-11-21T03:24:21,528 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1ceb52ec0d942ff87cf89b983117b4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732159460635 2024-11-21T03:24:21,528 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 87d71e6cc0c34579b1176717e9564071, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732159460635 2024-11-21T03:24:21,544 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#B#compaction#72 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:21,547 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:21,547 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/b01e6cc5cb054c82b1558db2106648f5 is 50, key is test_row_0/A:col10/1732159460970/Put/seqid=0 2024-11-21T03:24:21,548 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/38222a2428e54002bb64533a02519c9f is 50, key is test_row_0/B:col10/1732159460970/Put/seqid=0 2024-11-21T03:24:21,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741910_1086 (size=13119) 2024-11-21T03:24:21,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741909_1085 (size=13119) 2024-11-21T03:24:21,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T03:24:21,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:21,623 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T03:24:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:24:21,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/966d413561cd4cc692531b7813371cd5 is 50, key is test_row_0/A:col10/1732159461030/Put/seqid=0 2024-11-21T03:24:21,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:21,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741911_1087 (size=12301) 2024-11-21T03:24:21,687 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/966d413561cd4cc692531b7813371cd5 2024-11-21T03:24:21,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/3ac6b87546d74c78ab05ec0e9da329dd is 50, key is test_row_0/B:col10/1732159461030/Put/seqid=0 2024-11-21T03:24:21,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741912_1088 (size=12301) 2024-11-21T03:24:21,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159521802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:21,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159521808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:21,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159521918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159521919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:21,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:24:21,979 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/38222a2428e54002bb64533a02519c9f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/38222a2428e54002bb64533a02519c9f 2024-11-21T03:24:21,986 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aebdff1855164d839f0828203eeecd04/B of aebdff1855164d839f0828203eeecd04 into 38222a2428e54002bb64533a02519c9f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:21,986 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:21,986 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/B, priority=12, startTime=1732159461524; duration=0sec 2024-11-21T03:24:21,986 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:21,986 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:B 2024-11-21T03:24:21,986 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:21,992 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:21,992 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/C is initiating minor compaction (all files) 2024-11-21T03:24:21,992 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/C in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:21,992 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/b800e00fe2344982b793affbdfaa9f3d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/717fd5735dbc4ae686f15cd6e8279bf4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/72c47b93b915462a90b753b1330ef39e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8fb6d181ebba4a6ea88b71b16f431df0] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=48.7 K 2024-11-21T03:24:21,993 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting b800e00fe2344982b793affbdfaa9f3d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732159458056 2024-11-21T03:24:21,994 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 717fd5735dbc4ae686f15cd6e8279bf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732159458749 2024-11-21T03:24:21,994 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c47b93b915462a90b753b1330ef39e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732159459451 2024-11-21T03:24:21,995 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb6d181ebba4a6ea88b71b16f431df0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732159460635 2024-11-21T03:24:22,015 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/b01e6cc5cb054c82b1558db2106648f5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/b01e6cc5cb054c82b1558db2106648f5 2024-11-21T03:24:22,031 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aebdff1855164d839f0828203eeecd04/A of aebdff1855164d839f0828203eeecd04 into b01e6cc5cb054c82b1558db2106648f5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:22,032 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:22,032 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/A, priority=12, startTime=1732159461523; duration=0sec 2024-11-21T03:24:22,032 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:22,032 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:A 2024-11-21T03:24:22,038 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#C#compaction#76 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:22,039 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/d97dec07c21e4527853d84e257dc4c5a is 50, key is test_row_0/C:col10/1732159460970/Put/seqid=0 2024-11-21T03:24:22,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741913_1089 (size=13119) 2024-11-21T03:24:22,052 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/d97dec07c21e4527853d84e257dc4c5a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d97dec07c21e4527853d84e257dc4c5a 2024-11-21T03:24:22,063 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aebdff1855164d839f0828203eeecd04/C of aebdff1855164d839f0828203eeecd04 into d97dec07c21e4527853d84e257dc4c5a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:22,064 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:22,064 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/C, priority=12, startTime=1732159461524; duration=0sec 2024-11-21T03:24:22,064 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:22,064 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:C 2024-11-21T03:24:22,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159522129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:22,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159522130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:22,163 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/3ac6b87546d74c78ab05ec0e9da329dd 2024-11-21T03:24:22,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/805f5968fd0f45bbb43343ef3c129676 is 50, key is test_row_0/C:col10/1732159461030/Put/seqid=0 2024-11-21T03:24:22,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741914_1090 (size=12301) 2024-11-21T03:24:22,200 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/805f5968fd0f45bbb43343ef3c129676 2024-11-21T03:24:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/966d413561cd4cc692531b7813371cd5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/966d413561cd4cc692531b7813371cd5 2024-11-21T03:24:22,219 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/966d413561cd4cc692531b7813371cd5, entries=150, sequenceid=371, filesize=12.0 K 2024-11-21T03:24:22,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/3ac6b87546d74c78ab05ec0e9da329dd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3ac6b87546d74c78ab05ec0e9da329dd 2024-11-21T03:24:22,232 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3ac6b87546d74c78ab05ec0e9da329dd, entries=150, sequenceid=371, filesize=12.0 K 2024-11-21T03:24:22,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/805f5968fd0f45bbb43343ef3c129676 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/805f5968fd0f45bbb43343ef3c129676 2024-11-21T03:24:22,247 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/805f5968fd0f45bbb43343ef3c129676, entries=150, sequenceid=371, filesize=12.0 K 2024-11-21T03:24:22,248 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for aebdff1855164d839f0828203eeecd04 in 625ms, sequenceid=371, compaction requested=false 2024-11-21T03:24:22,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:22,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:22,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-21T03:24:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-21T03:24:22,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-21T03:24:22,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 939 msec 2024-11-21T03:24:22,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 953 msec 2024-11-21T03:24:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:22,441 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:22,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T03:24:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:24:22,449 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:22,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-21T03:24:22,452 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:24:22,453 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:22,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/602f1ccbedd2459ebc17556dba07e034 is 50, key is test_row_0/A:col10/1732159462440/Put/seqid=0 2024-11-21T03:24:22,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:22,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159522476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:22,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159522477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:22,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741915_1091 (size=14741) 2024-11-21T03:24:22,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/602f1ccbedd2459ebc17556dba07e034 2024-11-21T03:24:22,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/4f838728df73426f9dd8417c7ceef915 is 50, key is test_row_0/B:col10/1732159462440/Put/seqid=0 2024-11-21T03:24:22,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741916_1092 (size=12301) 2024-11-21T03:24:22,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/4f838728df73426f9dd8417c7ceef915 2024-11-21T03:24:22,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/278652c7ae9b4161948501ef8e72c762 is 50, key is test_row_0/C:col10/1732159462440/Put/seqid=0 2024-11-21T03:24:22,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741917_1093 (size=12301) 2024-11-21T03:24:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:24:22,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159522588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:22,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159522588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:22,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T03:24:22,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:22,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:22,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:22,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:22,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:22,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T03:24:22,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:22,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:22,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:22,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:22,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:24:22,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159522798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:22,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159522798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:22,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T03:24:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:22,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:22,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/278652c7ae9b4161948501ef8e72c762 2024-11-21T03:24:22,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/602f1ccbedd2459ebc17556dba07e034 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/602f1ccbedd2459ebc17556dba07e034 2024-11-21T03:24:22,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/602f1ccbedd2459ebc17556dba07e034, entries=200, sequenceid=395, filesize=14.4 K 2024-11-21T03:24:22,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/4f838728df73426f9dd8417c7ceef915 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/4f838728df73426f9dd8417c7ceef915 2024-11-21T03:24:22,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/4f838728df73426f9dd8417c7ceef915, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T03:24:22,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/278652c7ae9b4161948501ef8e72c762 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/278652c7ae9b4161948501ef8e72c762 2024-11-21T03:24:22,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/278652c7ae9b4161948501ef8e72c762, entries=150, sequenceid=395, filesize=12.0 K 2024-11-21T03:24:22,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for aebdff1855164d839f0828203eeecd04 in 525ms, sequenceid=395, compaction requested=true 2024-11-21T03:24:22,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:22,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:22,966 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:22,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:22,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:22,967 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:22,968 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:22,968 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/A is initiating minor compaction (all files) 2024-11-21T03:24:22,968 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/A in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:22,969 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/b01e6cc5cb054c82b1558db2106648f5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/966d413561cd4cc692531b7813371cd5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/602f1ccbedd2459ebc17556dba07e034] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=39.2 K 2024-11-21T03:24:22,969 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:22,969 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/B is initiating minor compaction (all files) 2024-11-21T03:24:22,969 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/B in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:22,969 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/38222a2428e54002bb64533a02519c9f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3ac6b87546d74c78ab05ec0e9da329dd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/4f838728df73426f9dd8417c7ceef915] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.8 K 2024-11-21T03:24:22,970 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b01e6cc5cb054c82b1558db2106648f5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732159460635 2024-11-21T03:24:22,970 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 38222a2428e54002bb64533a02519c9f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732159460635 2024-11-21T03:24:22,970 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 966d413561cd4cc692531b7813371cd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732159461011 2024-11-21T03:24:22,970 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ac6b87546d74c78ab05ec0e9da329dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732159461011 2024-11-21T03:24:22,971 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 602f1ccbedd2459ebc17556dba07e034, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732159461797 2024-11-21T03:24:22,971 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f838728df73426f9dd8417c7ceef915, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732159461797 2024-11-21T03:24:22,991 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:22,992 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/43395cbd921443e194c189e7addeda14 is 50, key is test_row_0/B:col10/1732159462440/Put/seqid=0 2024-11-21T03:24:22,996 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#A#compaction#82 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:22,997 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e35401973f134e4c80373c6794b84f38 is 50, key is test_row_0/A:col10/1732159462440/Put/seqid=0 2024-11-21T03:24:23,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741918_1094 (size=13221) 2024-11-21T03:24:23,019 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/43395cbd921443e194c189e7addeda14 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/43395cbd921443e194c189e7addeda14 2024-11-21T03:24:23,028 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/B of aebdff1855164d839f0828203eeecd04 into 43395cbd921443e194c189e7addeda14(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:23,028 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:23,029 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/B, priority=13, startTime=1732159462966; duration=0sec 2024-11-21T03:24:23,029 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:23,029 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:B 2024-11-21T03:24:23,029 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:23,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741919_1095 (size=13221) 2024-11-21T03:24:23,031 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:23,032 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/C is initiating minor compaction (all files) 2024-11-21T03:24:23,033 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/C in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:23,034 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d97dec07c21e4527853d84e257dc4c5a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/805f5968fd0f45bbb43343ef3c129676, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/278652c7ae9b4161948501ef8e72c762] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.8 K 2024-11-21T03:24:23,035 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d97dec07c21e4527853d84e257dc4c5a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732159460635 2024-11-21T03:24:23,037 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 805f5968fd0f45bbb43343ef3c129676, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732159461011 2024-11-21T03:24:23,037 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 278652c7ae9b4161948501ef8e72c762, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732159461797 2024-11-21T03:24:23,044 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e35401973f134e4c80373c6794b84f38 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e35401973f134e4c80373c6794b84f38 2024-11-21T03:24:23,058 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/A of aebdff1855164d839f0828203eeecd04 into e35401973f134e4c80373c6794b84f38(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:23,058 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:23,058 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/A, priority=13, startTime=1732159462966; duration=0sec 2024-11-21T03:24:23,058 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:23,058 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:A 2024-11-21T03:24:23,071 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#C#compaction#83 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:23,071 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/a5eb1824185d447a93d4b944e3bfef9d is 50, key is test_row_0/C:col10/1732159462440/Put/seqid=0 2024-11-21T03:24:23,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T03:24:23,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:23,075 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T03:24:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:24:23,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/c9f6ad6b5e3149dcaf7d9db7e84cf6b2 is 50, key is test_row_0/A:col10/1732159462468/Put/seqid=0 2024-11-21T03:24:23,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741920_1096 (size=13221) 2024-11-21T03:24:23,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741921_1097 (size=12301) 2024-11-21T03:24:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:23,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:23,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:23,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159523161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159523162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:23,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159523268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:23,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159523277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159523478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:23,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159523487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:23,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:23,494 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/a5eb1824185d447a93d4b944e3bfef9d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a5eb1824185d447a93d4b944e3bfef9d 2024-11-21T03:24:23,497 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/c9f6ad6b5e3149dcaf7d9db7e84cf6b2 2024-11-21T03:24:23,502 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/C of aebdff1855164d839f0828203eeecd04 into a5eb1824185d447a93d4b944e3bfef9d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:23,502 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:23,502 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/C, priority=13, startTime=1732159462967; duration=0sec 2024-11-21T03:24:23,502 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:23,502 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:C 2024-11-21T03:24:23,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/0f660be1ad2748a3b050edfb7e0937ec is 50, key is test_row_0/B:col10/1732159462468/Put/seqid=0 2024-11-21T03:24:23,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741922_1098 (size=12301) 2024-11-21T03:24:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:24:23,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159523788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:23,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159523797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:23,919 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/0f660be1ad2748a3b050edfb7e0937ec 2024-11-21T03:24:23,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/87f0a83f3edc40ca9533c134ff2bf01b is 50, key is test_row_0/C:col10/1732159462468/Put/seqid=0 2024-11-21T03:24:23,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741923_1099 (size=12301) 2024-11-21T03:24:23,952 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/87f0a83f3edc40ca9533c134ff2bf01b 2024-11-21T03:24:23,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/c9f6ad6b5e3149dcaf7d9db7e84cf6b2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c9f6ad6b5e3149dcaf7d9db7e84cf6b2 2024-11-21T03:24:23,971 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c9f6ad6b5e3149dcaf7d9db7e84cf6b2, entries=150, sequenceid=412, filesize=12.0 K 2024-11-21T03:24:23,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/0f660be1ad2748a3b050edfb7e0937ec as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0f660be1ad2748a3b050edfb7e0937ec 2024-11-21T03:24:23,982 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0f660be1ad2748a3b050edfb7e0937ec, entries=150, sequenceid=412, filesize=12.0 K 2024-11-21T03:24:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/87f0a83f3edc40ca9533c134ff2bf01b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/87f0a83f3edc40ca9533c134ff2bf01b 2024-11-21T03:24:23,992 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/87f0a83f3edc40ca9533c134ff2bf01b, entries=150, sequenceid=412, filesize=12.0 K 2024-11-21T03:24:23,993 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for aebdff1855164d839f0828203eeecd04 in 919ms, sequenceid=412, compaction requested=false 2024-11-21T03:24:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-21T03:24:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-21T03:24:23,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-21T03:24:23,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5430 sec 2024-11-21T03:24:24,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.5510 sec 2024-11-21T03:24:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:24,120 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:24,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:24:24,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/75a13bf43ca940ee8976e4ad09308480 is 50, key is test_row_0/A:col10/1732159463159/Put/seqid=0 2024-11-21T03:24:24,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741924_1100 (size=12301) 2024-11-21T03:24:24,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159524153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159524154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159524156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159524268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159524268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159524268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159524308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159524308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159524478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159524478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159524479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/75a13bf43ca940ee8976e4ad09308480 2024-11-21T03:24:24,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/914d1669a65a4d81a6608ea9404c1da3 is 50, key is test_row_0/B:col10/1732159463159/Put/seqid=0 2024-11-21T03:24:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:24:24,598 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:24,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-21T03:24:24,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741925_1101 (size=12301) 2024-11-21T03:24:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:24:24,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/914d1669a65a4d81a6608ea9404c1da3 2024-11-21T03:24:24,603 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:24,606 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:24,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:24,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/d4e9a431cad0454986eabda6f9a073d7 is 50, key is test_row_0/C:col10/1732159463159/Put/seqid=0 2024-11-21T03:24:24,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741926_1102 (size=12301) 2024-11-21T03:24:24,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/d4e9a431cad0454986eabda6f9a073d7 2024-11-21T03:24:24,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/75a13bf43ca940ee8976e4ad09308480 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/75a13bf43ca940ee8976e4ad09308480 2024-11-21T03:24:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:24:24,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/75a13bf43ca940ee8976e4ad09308480, entries=150, sequenceid=436, filesize=12.0 K 2024-11-21T03:24:24,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/914d1669a65a4d81a6608ea9404c1da3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/914d1669a65a4d81a6608ea9404c1da3 2024-11-21T03:24:24,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/914d1669a65a4d81a6608ea9404c1da3, entries=150, sequenceid=436, filesize=12.0 K 2024-11-21T03:24:24,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/d4e9a431cad0454986eabda6f9a073d7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d4e9a431cad0454986eabda6f9a073d7 2024-11-21T03:24:24,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d4e9a431cad0454986eabda6f9a073d7, entries=150, sequenceid=436, filesize=12.0 K 2024-11-21T03:24:24,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for aebdff1855164d839f0828203eeecd04 in 629ms, sequenceid=436, compaction requested=true 2024-11-21T03:24:24,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:24,749 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:24,750 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:24,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:24,752 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:24,752 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:24,752 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/A is initiating minor compaction (all files) 2024-11-21T03:24:24,752 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/B is initiating minor compaction (all files) 2024-11-21T03:24:24,752 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/B in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:24,752 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/A in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:24,752 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e35401973f134e4c80373c6794b84f38, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c9f6ad6b5e3149dcaf7d9db7e84cf6b2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/75a13bf43ca940ee8976e4ad09308480] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.9 K 2024-11-21T03:24:24,752 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/43395cbd921443e194c189e7addeda14, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0f660be1ad2748a3b050edfb7e0937ec, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/914d1669a65a4d81a6608ea9404c1da3] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.9 K 2024-11-21T03:24:24,753 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e35401973f134e4c80373c6794b84f38, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732159461797 2024-11-21T03:24:24,753 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 43395cbd921443e194c189e7addeda14, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732159461797 2024-11-21T03:24:24,754 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f660be1ad2748a3b050edfb7e0937ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732159462468 2024-11-21T03:24:24,754 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9f6ad6b5e3149dcaf7d9db7e84cf6b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732159462468 2024-11-21T03:24:24,754 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 914d1669a65a4d81a6608ea9404c1da3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732159463159 2024-11-21T03:24:24,754 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75a13bf43ca940ee8976e4ad09308480, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732159463159 2024-11-21T03:24:24,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T03:24:24,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:24,762 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:24:24,767 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:24,768 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/0be3b04399f84a6399415288bec103ec is 50, key is test_row_0/A:col10/1732159463159/Put/seqid=0 2024-11-21T03:24:24,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/dec0f70e77654f94a859edfd2e501bda is 50, key is test_row_0/A:col10/1732159464151/Put/seqid=0 2024-11-21T03:24:24,776 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#B#compaction#91 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:24,777 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/8a5e98d1761c4f269648a214eb2ada78 is 50, key is test_row_0/B:col10/1732159463159/Put/seqid=0 2024-11-21T03:24:24,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741927_1103 (size=13323) 2024-11-21T03:24:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:24,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:24,806 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/0be3b04399f84a6399415288bec103ec as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/0be3b04399f84a6399415288bec103ec 2024-11-21T03:24:24,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741928_1104 (size=12301) 2024-11-21T03:24:24,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741929_1105 (size=13323) 2024-11-21T03:24:24,821 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/A of aebdff1855164d839f0828203eeecd04 into 0be3b04399f84a6399415288bec103ec(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:24,821 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:24,821 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/A, priority=13, startTime=1732159464749; duration=0sec 2024-11-21T03:24:24,821 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:24,821 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:A 2024-11-21T03:24:24,821 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:24,823 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:24,823 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): aebdff1855164d839f0828203eeecd04/C is initiating minor compaction (all files) 2024-11-21T03:24:24,823 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aebdff1855164d839f0828203eeecd04/C in TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:24,823 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a5eb1824185d447a93d4b944e3bfef9d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/87f0a83f3edc40ca9533c134ff2bf01b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d4e9a431cad0454986eabda6f9a073d7] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp, totalSize=36.9 K 2024-11-21T03:24:24,825 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5eb1824185d447a93d4b944e3bfef9d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732159461797 2024-11-21T03:24:24,825 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87f0a83f3edc40ca9533c134ff2bf01b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732159462468 2024-11-21T03:24:24,826 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4e9a431cad0454986eabda6f9a073d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732159463159 2024-11-21T03:24:24,846 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aebdff1855164d839f0828203eeecd04#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:24,846 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/4c9b6d3a8a834fe59362b672a0e70d19 is 50, key is test_row_0/C:col10/1732159463159/Put/seqid=0 2024-11-21T03:24:24,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159524844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159524849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159524849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741930_1106 (size=13323) 2024-11-21T03:24:24,876 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/4c9b6d3a8a834fe59362b672a0e70d19 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/4c9b6d3a8a834fe59362b672a0e70d19 2024-11-21T03:24:24,895 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/C of aebdff1855164d839f0828203eeecd04 into 4c9b6d3a8a834fe59362b672a0e70d19(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:24,895 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:24,895 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/C, priority=13, startTime=1732159464750; duration=0sec 2024-11-21T03:24:24,896 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:24,896 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:C 2024-11-21T03:24:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:24:24,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159524958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159524958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:24,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159524967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:25,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159525168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:25,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159525168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:25,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159525177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:25,210 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/dec0f70e77654f94a859edfd2e501bda 2024-11-21T03:24:25,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/a942fb0b06c74817baf37f95ed476fbc is 50, key is test_row_0/B:col10/1732159464151/Put/seqid=0 2024-11-21T03:24:25,228 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/8a5e98d1761c4f269648a214eb2ada78 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/8a5e98d1761c4f269648a214eb2ada78 2024-11-21T03:24:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:24:25,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741931_1107 (size=12301) 2024-11-21T03:24:25,233 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/a942fb0b06c74817baf37f95ed476fbc 2024-11-21T03:24:25,237 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aebdff1855164d839f0828203eeecd04/B of aebdff1855164d839f0828203eeecd04 into 8a5e98d1761c4f269648a214eb2ada78(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:25,238 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:25,238 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., storeName=aebdff1855164d839f0828203eeecd04/B, priority=13, startTime=1732159464750; duration=0sec 2024-11-21T03:24:25,238 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:25,238 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:B 2024-11-21T03:24:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/f1bc63da66b24cdd93ee479b820f1c01 is 50, key is test_row_0/C:col10/1732159464151/Put/seqid=0 2024-11-21T03:24:25,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741932_1108 (size=12301) 2024-11-21T03:24:25,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53600 deadline: 1732159525318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:25,319 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_1' in aebdff1855164d839f0828203eeecd04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:25,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53614 deadline: 1732159525318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:25,322 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_0' in aebdff1855164d839f0828203eeecd04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:25,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:25,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159525478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159525478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:25,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159525488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:25,663 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/f1bc63da66b24cdd93ee479b820f1c01 2024-11-21T03:24:25,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/dec0f70e77654f94a859edfd2e501bda as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/dec0f70e77654f94a859edfd2e501bda 2024-11-21T03:24:25,682 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/dec0f70e77654f94a859edfd2e501bda, entries=150, sequenceid=449, filesize=12.0 K 2024-11-21T03:24:25,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/a942fb0b06c74817baf37f95ed476fbc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/a942fb0b06c74817baf37f95ed476fbc 2024-11-21T03:24:25,690 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/a942fb0b06c74817baf37f95ed476fbc, entries=150, sequenceid=449, filesize=12.0 K 2024-11-21T03:24:25,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/f1bc63da66b24cdd93ee479b820f1c01 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/f1bc63da66b24cdd93ee479b820f1c01 2024-11-21T03:24:25,700 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/f1bc63da66b24cdd93ee479b820f1c01, entries=150, sequenceid=449, filesize=12.0 K 2024-11-21T03:24:25,701 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for aebdff1855164d839f0828203eeecd04 in 939ms, sequenceid=449, compaction requested=false 2024-11-21T03:24:25,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:25,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:25,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-21T03:24:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-21T03:24:25,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-21T03:24:25,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0970 sec 2024-11-21T03:24:25,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.1080 sec 2024-11-21T03:24:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:24:25,738 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:25,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-21T03:24:25,741 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T03:24:25,742 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:25,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T03:24:25,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T03:24:25,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:25,898 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:24:25,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e2059ea0d6214dac9c61897671267201 is 50, key is test_row_0/A:col10/1732159464840/Put/seqid=0 2024-11-21T03:24:25,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741933_1109 (size=12301) 2024-11-21T03:24:25,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. as already flushing 2024-11-21T03:24:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:26,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159526012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159526014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159526018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T03:24:26,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159526128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159526128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159526137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,327 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e2059ea0d6214dac9c61897671267201 2024-11-21T03:24:26,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/ae814aed040045118168a042cdd67610 is 50, key is test_row_0/B:col10/1732159464840/Put/seqid=0 2024-11-21T03:24:26,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159526337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159526337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741934_1110 (size=12301) 2024-11-21T03:24:26,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159526347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T03:24:26,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159526647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159526647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159526657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:26,750 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/ae814aed040045118168a042cdd67610 2024-11-21T03:24:26,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/81514e67f2e2451296025ebebbbb6bc6 is 50, key is test_row_0/C:col10/1732159464840/Put/seqid=0 2024-11-21T03:24:26,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741935_1111 (size=12301) 2024-11-21T03:24:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T03:24:27,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53558 deadline: 1732159527158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:27,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:27,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:27,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:27,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53584 deadline: 1732159527168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:27,172 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/81514e67f2e2451296025ebebbbb6bc6 2024-11-21T03:24:27,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e2059ea0d6214dac9c61897671267201 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e2059ea0d6214dac9c61897671267201 2024-11-21T03:24:27,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53554 deadline: 1732159527181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:27,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:24:27,187 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e2059ea0d6214dac9c61897671267201, entries=150, sequenceid=476, filesize=12.0 K 2024-11-21T03:24:27,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/ae814aed040045118168a042cdd67610 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/ae814aed040045118168a042cdd67610 2024-11-21T03:24:27,197 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/ae814aed040045118168a042cdd67610, entries=150, sequenceid=476, filesize=12.0 K 2024-11-21T03:24:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/81514e67f2e2451296025ebebbbb6bc6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/81514e67f2e2451296025ebebbbb6bc6 2024-11-21T03:24:27,208 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/81514e67f2e2451296025ebebbbb6bc6, entries=150, sequenceid=476, filesize=12.0 K 2024-11-21T03:24:27,209 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for aebdff1855164d839f0828203eeecd04 in 1311ms, sequenceid=476, compaction requested=true 2024-11-21T03:24:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-21T03:24:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-21T03:24:27,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-21T03:24:27,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4680 sec 2024-11-21T03:24:27,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4750 sec 2024-11-21T03:24:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:27,332 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:27,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:24:27,333 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-21T03:24:27,333 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-21T03:24:27,333 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:27,333 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:27,333 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-21T03:24:27,333 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:27,334 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-21T03:24:27,334 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:27,335 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-21T03:24:27,335 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:27,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e544e01ab0964bceaf3c4b01f194ffb4 is 50, key is test_row_0/A:col10/1732159467330/Put/seqid=0 2024-11-21T03:24:27,339 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,339 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-21T03:24:27,340 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:27,340 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,340 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,340 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:27,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:27,340 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:27,340 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,340 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,340 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:27,341 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,341 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:27,341 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,341 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:27,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:27,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741936_1112 (size=12301) 2024-11-21T03:24:27,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e544e01ab0964bceaf3c4b01f194ffb4 2024-11-21T03:24:27,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/79935a64ea60419a804dbbb29e342628 is 50, key is test_row_0/B:col10/1732159467330/Put/seqid=0 2024-11-21T03:24:27,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741937_1113 (size=12301) 2024-11-21T03:24:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T03:24:27,889 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:28,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/79935a64ea60419a804dbbb29e342628 2024-11-21T03:24:28,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/7f9b95570b73439eb8e9bfd0386a9350 is 50, key is test_row_0/C:col10/1732159467330/Put/seqid=0 2024-11-21T03:24:28,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741938_1114 (size=12301) 2024-11-21T03:24:28,179 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-21T03:24:28,179 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-21T03:24:28,179 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:28,179 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:28,179 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:28,179 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:28,179 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:28,179 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:28,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:28,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:28,199 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-21T03:24:28,199 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:28,199 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:28,199 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:28,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:28,199 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2571 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2546 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1224 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3669 rows 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1217 2024-11-21T03:24:28,200 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3648 rows 2024-11-21T03:24:28,200 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:24:28,200 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:24:28,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:28,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:28,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:28,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T03:24:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:28,211 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159468211"}]},"ts":"1732159468211"} 2024-11-21T03:24:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T03:24:28,213 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T03:24:28,214 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T03:24:28,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:24:28,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aebdff1855164d839f0828203eeecd04, UNASSIGN}] 2024-11-21T03:24:28,220 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aebdff1855164d839f0828203eeecd04, UNASSIGN 2024-11-21T03:24:28,221 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=aebdff1855164d839f0828203eeecd04, regionState=CLOSING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:28,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aebdff1855164d839f0828203eeecd04, UNASSIGN because future has completed 2024-11-21T03:24:28,223 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:24:28,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:24:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T03:24:28,380 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:28,381 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:24:28,381 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing aebdff1855164d839f0828203eeecd04, disabling compactions & flushes 2024-11-21T03:24:28,381 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T03:24:28,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/7f9b95570b73439eb8e9bfd0386a9350 2024-11-21T03:24:28,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/e544e01ab0964bceaf3c4b01f194ffb4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e544e01ab0964bceaf3c4b01f194ffb4 2024-11-21T03:24:28,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e544e01ab0964bceaf3c4b01f194ffb4, entries=150, sequenceid=489, filesize=12.0 K 2024-11-21T03:24:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/79935a64ea60419a804dbbb29e342628 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/79935a64ea60419a804dbbb29e342628 2024-11-21T03:24:28,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/79935a64ea60419a804dbbb29e342628, entries=150, sequenceid=489, filesize=12.0 K 2024-11-21T03:24:28,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/7f9b95570b73439eb8e9bfd0386a9350 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7f9b95570b73439eb8e9bfd0386a9350 2024-11-21T03:24:28,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7f9b95570b73439eb8e9bfd0386a9350, entries=150, sequenceid=489, filesize=12.0 K 2024-11-21T03:24:28,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=26.84 KB/27480 for aebdff1855164d839f0828203eeecd04 in 1266ms, sequenceid=489, compaction requested=true 2024-11-21T03:24:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aebdff1855164d839f0828203eeecd04: 2024-11-21T03:24:28,598 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:28,598 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:28,598 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. after waiting 0 ms 2024-11-21T03:24:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:28,598 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:28,598 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. because compaction request was cancelled 2024-11-21T03:24:28,598 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. because compaction request was cancelled 2024-11-21T03:24:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:28,598 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:B 2024-11-21T03:24:28,598 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:A 2024-11-21T03:24:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aebdff1855164d839f0828203eeecd04:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:28,598 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. because compaction request was cancelled 2024-11-21T03:24:28,598 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aebdff1855164d839f0828203eeecd04:C 2024-11-21T03:24:28,598 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing aebdff1855164d839f0828203eeecd04 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-21T03:24:28,604 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/a24ee87682d941c48f1e4e9e0b8d456f is 50, key is test_row_0/A:col10/1732159468198/Put/seqid=0 2024-11-21T03:24:28,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741939_1115 (size=12301) 2024-11-21T03:24:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T03:24:29,009 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/a24ee87682d941c48f1e4e9e0b8d456f 2024-11-21T03:24:29,019 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/9ff41fde02f543a38d9c43816f87bcf9 is 50, key is test_row_0/B:col10/1732159468198/Put/seqid=0 2024-11-21T03:24:29,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741940_1116 (size=12301) 2024-11-21T03:24:29,317 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T03:24:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T03:24:29,444 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/9ff41fde02f543a38d9c43816f87bcf9 2024-11-21T03:24:29,459 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/c47f573254404d0891127e54599be4d5 is 50, key is test_row_0/C:col10/1732159468198/Put/seqid=0 2024-11-21T03:24:29,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741941_1117 (size=12301) 2024-11-21T03:24:29,864 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/c47f573254404d0891127e54599be4d5 2024-11-21T03:24:29,871 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/A/a24ee87682d941c48f1e4e9e0b8d456f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/a24ee87682d941c48f1e4e9e0b8d456f 2024-11-21T03:24:29,877 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/a24ee87682d941c48f1e4e9e0b8d456f, entries=150, sequenceid=496, filesize=12.0 K 2024-11-21T03:24:29,878 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/B/9ff41fde02f543a38d9c43816f87bcf9 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/9ff41fde02f543a38d9c43816f87bcf9 2024-11-21T03:24:29,883 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/9ff41fde02f543a38d9c43816f87bcf9, entries=150, sequenceid=496, filesize=12.0 K 2024-11-21T03:24:29,885 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/.tmp/C/c47f573254404d0891127e54599be4d5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/c47f573254404d0891127e54599be4d5 2024-11-21T03:24:29,891 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/c47f573254404d0891127e54599be4d5, entries=150, sequenceid=496, filesize=12.0 K 2024-11-21T03:24:29,892 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for aebdff1855164d839f0828203eeecd04 in 1294ms, sequenceid=496, compaction requested=true 2024-11-21T03:24:29,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/67cf36c192964c0385926183733b7154, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/853126dcf673489980680fed8925142d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/5db2bda6ec9841c0b09e891177422ec6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c537c84544204758ad3703f4ceb3a914, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/2df9596da1f3451b80cbe0af83aa4ca4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/ddfcb7dafa994cdf9848d4854d358445, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/f78abce5aadc4da0b125c4620f2e7faa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6f815b1f9b544f2e99eb5f40a18f5077, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/972488cc9928452aa44da13de0228bb3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/a53394f138a54eb28b00a2203d250231, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/29d6721902384e29b0ba794f6feb012f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/8720cb8ee47b44e4a9e85c3764eaf829, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6dc76ed0f02c4f05abbde0db265cd57b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/abfc7e0f43334b9bb8777e53884b5ba7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/405fa82743ef41659cf05e4e8f8678de, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cba9df26be5d4ef6abf445953f27c4cd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/11985f7157d544e89461ae0b3c57946d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/20d03db84d9f45fda09f1fdb8cf99f88, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e5cba214774641e3b39732d40af497f1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/d6d2e66be4ec4af880da9500cb895855, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/41a69cde80954041b50011dceff21e01, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/873113d059fd45bda78b806809c4aaa3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cad09c118a7d4fb4a26247696713c8db, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/b01e6cc5cb054c82b1558db2106648f5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c1ceb52ec0d942ff87cf89b983117b4f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/966d413561cd4cc692531b7813371cd5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/602f1ccbedd2459ebc17556dba07e034, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e35401973f134e4c80373c6794b84f38, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c9f6ad6b5e3149dcaf7d9db7e84cf6b2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/75a13bf43ca940ee8976e4ad09308480] to archive 2024-11-21T03:24:29,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:24:29,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/67cf36c192964c0385926183733b7154 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/67cf36c192964c0385926183733b7154 2024-11-21T03:24:29,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/853126dcf673489980680fed8925142d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/853126dcf673489980680fed8925142d 2024-11-21T03:24:29,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/5db2bda6ec9841c0b09e891177422ec6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/5db2bda6ec9841c0b09e891177422ec6 2024-11-21T03:24:29,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c537c84544204758ad3703f4ceb3a914 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c537c84544204758ad3703f4ceb3a914 2024-11-21T03:24:29,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/2df9596da1f3451b80cbe0af83aa4ca4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/2df9596da1f3451b80cbe0af83aa4ca4 2024-11-21T03:24:29,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/ddfcb7dafa994cdf9848d4854d358445 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/ddfcb7dafa994cdf9848d4854d358445 2024-11-21T03:24:29,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/f78abce5aadc4da0b125c4620f2e7faa to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/f78abce5aadc4da0b125c4620f2e7faa 2024-11-21T03:24:29,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6f815b1f9b544f2e99eb5f40a18f5077 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6f815b1f9b544f2e99eb5f40a18f5077 2024-11-21T03:24:29,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/972488cc9928452aa44da13de0228bb3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/972488cc9928452aa44da13de0228bb3 2024-11-21T03:24:29,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/a53394f138a54eb28b00a2203d250231 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/a53394f138a54eb28b00a2203d250231 2024-11-21T03:24:29,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/29d6721902384e29b0ba794f6feb012f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/29d6721902384e29b0ba794f6feb012f 2024-11-21T03:24:29,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/8720cb8ee47b44e4a9e85c3764eaf829 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/8720cb8ee47b44e4a9e85c3764eaf829 2024-11-21T03:24:29,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6dc76ed0f02c4f05abbde0db265cd57b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/6dc76ed0f02c4f05abbde0db265cd57b 2024-11-21T03:24:29,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/abfc7e0f43334b9bb8777e53884b5ba7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/abfc7e0f43334b9bb8777e53884b5ba7 2024-11-21T03:24:29,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/405fa82743ef41659cf05e4e8f8678de to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/405fa82743ef41659cf05e4e8f8678de 2024-11-21T03:24:29,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cba9df26be5d4ef6abf445953f27c4cd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cba9df26be5d4ef6abf445953f27c4cd 2024-11-21T03:24:29,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/11985f7157d544e89461ae0b3c57946d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/11985f7157d544e89461ae0b3c57946d 2024-11-21T03:24:29,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/20d03db84d9f45fda09f1fdb8cf99f88 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/20d03db84d9f45fda09f1fdb8cf99f88 2024-11-21T03:24:29,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e5cba214774641e3b39732d40af497f1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e5cba214774641e3b39732d40af497f1 2024-11-21T03:24:29,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/d6d2e66be4ec4af880da9500cb895855 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/d6d2e66be4ec4af880da9500cb895855 2024-11-21T03:24:29,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/41a69cde80954041b50011dceff21e01 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/41a69cde80954041b50011dceff21e01 2024-11-21T03:24:29,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/873113d059fd45bda78b806809c4aaa3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/873113d059fd45bda78b806809c4aaa3 2024-11-21T03:24:29,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cad09c118a7d4fb4a26247696713c8db to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/cad09c118a7d4fb4a26247696713c8db 2024-11-21T03:24:29,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/b01e6cc5cb054c82b1558db2106648f5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/b01e6cc5cb054c82b1558db2106648f5 2024-11-21T03:24:29,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c1ceb52ec0d942ff87cf89b983117b4f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c1ceb52ec0d942ff87cf89b983117b4f 2024-11-21T03:24:29,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/966d413561cd4cc692531b7813371cd5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/966d413561cd4cc692531b7813371cd5 2024-11-21T03:24:29,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/602f1ccbedd2459ebc17556dba07e034 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/602f1ccbedd2459ebc17556dba07e034 2024-11-21T03:24:29,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e35401973f134e4c80373c6794b84f38 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e35401973f134e4c80373c6794b84f38 2024-11-21T03:24:29,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c9f6ad6b5e3149dcaf7d9db7e84cf6b2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/c9f6ad6b5e3149dcaf7d9db7e84cf6b2 2024-11-21T03:24:29,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/75a13bf43ca940ee8976e4ad09308480 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/75a13bf43ca940ee8976e4ad09308480 2024-11-21T03:24:29,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/bac35f613e60460889fcea94304eeff0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/64224194ccf34b3d86cf6c2f0e9ee9e7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f0fd693f5877443fbf1abaf2792a78ef, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/efac21f7460c4248a9f5e51ef1921199, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7f0bfb8f692148139682554069242fcd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/6b3ae2158ad24940884da6dbf8faf218, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3053ef400e784c23a6f68a9e3e6be62f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/8aa299cd56414bd5810e42fdaa17307d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d0f3b974f72741628c7a6442b207e334, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/a0e453dceb1d4bde80ae403b0686b4eb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/ccbea09c0a234aed866db64ca1d315cd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d7f4d8ca788e4eb6a3ac1e51bfa2024f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f1a8d6b2cb4d4ed5a99438bafe27f392, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/28c7a07d049f4310bdbe2ed9badf7cdd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/42136dddf2aa47bdb1567dab3d07fd5c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/e48c87ec53eb428ba287565a128023f9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d205f7e484da4eddb4b1d5535430a531, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/54510f14816e4e52b87cb13ea118dc4e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0deb3bf4ef4342eca8f6c052bd9abcfe, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/fff9c821101045ac9f90552f5aaac805, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7b1506433e25449bab53dbac9d715c5f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/c25fb1487295489eabac40523bce3443, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/5dabf226a4c544459f5fd40580659b2a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/38222a2428e54002bb64533a02519c9f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/87d71e6cc0c34579b1176717e9564071, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3ac6b87546d74c78ab05ec0e9da329dd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/43395cbd921443e194c189e7addeda14, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/4f838728df73426f9dd8417c7ceef915, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0f660be1ad2748a3b050edfb7e0937ec, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/914d1669a65a4d81a6608ea9404c1da3] to archive 2024-11-21T03:24:29,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:24:29,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/bac35f613e60460889fcea94304eeff0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/bac35f613e60460889fcea94304eeff0 2024-11-21T03:24:29,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/64224194ccf34b3d86cf6c2f0e9ee9e7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/64224194ccf34b3d86cf6c2f0e9ee9e7 2024-11-21T03:24:29,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f0fd693f5877443fbf1abaf2792a78ef to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f0fd693f5877443fbf1abaf2792a78ef 2024-11-21T03:24:29,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/efac21f7460c4248a9f5e51ef1921199 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/efac21f7460c4248a9f5e51ef1921199 2024-11-21T03:24:29,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7f0bfb8f692148139682554069242fcd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7f0bfb8f692148139682554069242fcd 2024-11-21T03:24:29,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/6b3ae2158ad24940884da6dbf8faf218 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/6b3ae2158ad24940884da6dbf8faf218 2024-11-21T03:24:29,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3053ef400e784c23a6f68a9e3e6be62f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3053ef400e784c23a6f68a9e3e6be62f 2024-11-21T03:24:29,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/8aa299cd56414bd5810e42fdaa17307d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/8aa299cd56414bd5810e42fdaa17307d 2024-11-21T03:24:29,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d0f3b974f72741628c7a6442b207e334 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d0f3b974f72741628c7a6442b207e334 2024-11-21T03:24:29,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/a0e453dceb1d4bde80ae403b0686b4eb to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/a0e453dceb1d4bde80ae403b0686b4eb 2024-11-21T03:24:29,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/ccbea09c0a234aed866db64ca1d315cd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/ccbea09c0a234aed866db64ca1d315cd 2024-11-21T03:24:29,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d7f4d8ca788e4eb6a3ac1e51bfa2024f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d7f4d8ca788e4eb6a3ac1e51bfa2024f 2024-11-21T03:24:29,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f1a8d6b2cb4d4ed5a99438bafe27f392 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/f1a8d6b2cb4d4ed5a99438bafe27f392 2024-11-21T03:24:29,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/28c7a07d049f4310bdbe2ed9badf7cdd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/28c7a07d049f4310bdbe2ed9badf7cdd 2024-11-21T03:24:29,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/42136dddf2aa47bdb1567dab3d07fd5c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/42136dddf2aa47bdb1567dab3d07fd5c 2024-11-21T03:24:29,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/e48c87ec53eb428ba287565a128023f9 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/e48c87ec53eb428ba287565a128023f9 2024-11-21T03:24:29,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d205f7e484da4eddb4b1d5535430a531 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/d205f7e484da4eddb4b1d5535430a531 2024-11-21T03:24:29,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/54510f14816e4e52b87cb13ea118dc4e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/54510f14816e4e52b87cb13ea118dc4e 2024-11-21T03:24:29,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0deb3bf4ef4342eca8f6c052bd9abcfe to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0deb3bf4ef4342eca8f6c052bd9abcfe 2024-11-21T03:24:29,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/fff9c821101045ac9f90552f5aaac805 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/fff9c821101045ac9f90552f5aaac805 2024-11-21T03:24:29,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7b1506433e25449bab53dbac9d715c5f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/7b1506433e25449bab53dbac9d715c5f 2024-11-21T03:24:29,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/c25fb1487295489eabac40523bce3443 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/c25fb1487295489eabac40523bce3443 2024-11-21T03:24:29,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/5dabf226a4c544459f5fd40580659b2a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/5dabf226a4c544459f5fd40580659b2a 2024-11-21T03:24:29,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/38222a2428e54002bb64533a02519c9f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/38222a2428e54002bb64533a02519c9f 2024-11-21T03:24:29,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/87d71e6cc0c34579b1176717e9564071 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/87d71e6cc0c34579b1176717e9564071 2024-11-21T03:24:29,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3ac6b87546d74c78ab05ec0e9da329dd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/3ac6b87546d74c78ab05ec0e9da329dd 2024-11-21T03:24:29,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/43395cbd921443e194c189e7addeda14 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/43395cbd921443e194c189e7addeda14 2024-11-21T03:24:29,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/4f838728df73426f9dd8417c7ceef915 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/4f838728df73426f9dd8417c7ceef915 2024-11-21T03:24:29,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0f660be1ad2748a3b050edfb7e0937ec to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/0f660be1ad2748a3b050edfb7e0937ec 2024-11-21T03:24:29,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/914d1669a65a4d81a6608ea9404c1da3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/914d1669a65a4d81a6608ea9404c1da3 2024-11-21T03:24:29,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/6ea237037f32450dafa22193dc02857b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a2011bd035204fa2bb860397e441284d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/47a092a6d79c4b8984645c521d9b67b7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/137a6a94db554fc19d5ba0ace55b5e90, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0c493afefcfc41c490b7209a67b53f8a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/41c21515cadf4fffa126cc9ba478bba2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/835971d78103471c850c62a9b62682aa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3f06283c2c6f4d3ba010c674772f9152, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3fd65325858a404e856e2bcc3238a1eb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/09d1751cef1e470d92580a16694e6ac8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/aa028d7449bd4612bde77a1b1a403b38, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7df0ddbd1f1e4dd4b6dd7b9d1631a4f0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/c49c5f321e8d480881a28694d564b673, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8c9bf3edc560403ca9c90d6a359e823f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/909a83b8189f496cb8279f612835c6c8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/ce9841549e634b9fa7e916d4c1c346bc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/48a81594b5aa40d8bd74a60deb6470f2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7eac5f4bd0ba4b5bafbbfde5ac52ae5c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0355cf2260ef45a8829a028f1f8a3e9f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/b800e00fe2344982b793affbdfaa9f3d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/97bec831e2c14e5a88c30601a467170a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/717fd5735dbc4ae686f15cd6e8279bf4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/72c47b93b915462a90b753b1330ef39e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d97dec07c21e4527853d84e257dc4c5a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8fb6d181ebba4a6ea88b71b16f431df0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/805f5968fd0f45bbb43343ef3c129676, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a5eb1824185d447a93d4b944e3bfef9d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/278652c7ae9b4161948501ef8e72c762, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/87f0a83f3edc40ca9533c134ff2bf01b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d4e9a431cad0454986eabda6f9a073d7] to archive 2024-11-21T03:24:29,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:24:29,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/6ea237037f32450dafa22193dc02857b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/6ea237037f32450dafa22193dc02857b 2024-11-21T03:24:29,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a2011bd035204fa2bb860397e441284d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a2011bd035204fa2bb860397e441284d 2024-11-21T03:24:29,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/47a092a6d79c4b8984645c521d9b67b7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/47a092a6d79c4b8984645c521d9b67b7 2024-11-21T03:24:29,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/137a6a94db554fc19d5ba0ace55b5e90 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/137a6a94db554fc19d5ba0ace55b5e90 2024-11-21T03:24:29,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0c493afefcfc41c490b7209a67b53f8a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0c493afefcfc41c490b7209a67b53f8a 2024-11-21T03:24:30,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/41c21515cadf4fffa126cc9ba478bba2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/41c21515cadf4fffa126cc9ba478bba2 2024-11-21T03:24:30,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/835971d78103471c850c62a9b62682aa to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/835971d78103471c850c62a9b62682aa 2024-11-21T03:24:30,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3f06283c2c6f4d3ba010c674772f9152 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3f06283c2c6f4d3ba010c674772f9152 2024-11-21T03:24:30,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3fd65325858a404e856e2bcc3238a1eb to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/3fd65325858a404e856e2bcc3238a1eb 2024-11-21T03:24:30,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/09d1751cef1e470d92580a16694e6ac8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/09d1751cef1e470d92580a16694e6ac8 2024-11-21T03:24:30,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/aa028d7449bd4612bde77a1b1a403b38 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/aa028d7449bd4612bde77a1b1a403b38 2024-11-21T03:24:30,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7df0ddbd1f1e4dd4b6dd7b9d1631a4f0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7df0ddbd1f1e4dd4b6dd7b9d1631a4f0 2024-11-21T03:24:30,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/c49c5f321e8d480881a28694d564b673 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/c49c5f321e8d480881a28694d564b673 2024-11-21T03:24:30,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8c9bf3edc560403ca9c90d6a359e823f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8c9bf3edc560403ca9c90d6a359e823f 2024-11-21T03:24:30,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/909a83b8189f496cb8279f612835c6c8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/909a83b8189f496cb8279f612835c6c8 2024-11-21T03:24:30,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/ce9841549e634b9fa7e916d4c1c346bc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/ce9841549e634b9fa7e916d4c1c346bc 2024-11-21T03:24:30,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/48a81594b5aa40d8bd74a60deb6470f2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/48a81594b5aa40d8bd74a60deb6470f2 2024-11-21T03:24:30,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7eac5f4bd0ba4b5bafbbfde5ac52ae5c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7eac5f4bd0ba4b5bafbbfde5ac52ae5c 2024-11-21T03:24:30,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0355cf2260ef45a8829a028f1f8a3e9f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/0355cf2260ef45a8829a028f1f8a3e9f 2024-11-21T03:24:30,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/b800e00fe2344982b793affbdfaa9f3d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/b800e00fe2344982b793affbdfaa9f3d 2024-11-21T03:24:30,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/97bec831e2c14e5a88c30601a467170a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/97bec831e2c14e5a88c30601a467170a 2024-11-21T03:24:30,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/717fd5735dbc4ae686f15cd6e8279bf4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/717fd5735dbc4ae686f15cd6e8279bf4 2024-11-21T03:24:30,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/72c47b93b915462a90b753b1330ef39e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/72c47b93b915462a90b753b1330ef39e 2024-11-21T03:24:30,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d97dec07c21e4527853d84e257dc4c5a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d97dec07c21e4527853d84e257dc4c5a 2024-11-21T03:24:30,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8fb6d181ebba4a6ea88b71b16f431df0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/8fb6d181ebba4a6ea88b71b16f431df0 2024-11-21T03:24:30,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/805f5968fd0f45bbb43343ef3c129676 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/805f5968fd0f45bbb43343ef3c129676 2024-11-21T03:24:30,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a5eb1824185d447a93d4b944e3bfef9d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/a5eb1824185d447a93d4b944e3bfef9d 2024-11-21T03:24:30,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/278652c7ae9b4161948501ef8e72c762 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/278652c7ae9b4161948501ef8e72c762 2024-11-21T03:24:30,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/87f0a83f3edc40ca9533c134ff2bf01b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/87f0a83f3edc40ca9533c134ff2bf01b 2024-11-21T03:24:30,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d4e9a431cad0454986eabda6f9a073d7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/d4e9a431cad0454986eabda6f9a073d7 2024-11-21T03:24:30,034 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/recovered.edits/499.seqid, newMaxSeqId=499, maxSeqId=1 2024-11-21T03:24:30,034 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. 2024-11-21T03:24:30,035 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for aebdff1855164d839f0828203eeecd04: Waiting for close lock at 1732159468381Running coprocessor pre-close hooks at 1732159468381Disabling compacts and flushes for region at 1732159468381Disabling writes for close at 1732159468598 (+217 ms)Obtaining lock to block concurrent updates at 1732159468599 (+1 ms)Preparing flush snapshotting stores in aebdff1855164d839f0828203eeecd04 at 1732159468599Finished memstore snapshotting TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732159468599Flushing stores of TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04. at 1732159468599Flushing aebdff1855164d839f0828203eeecd04/A: creating writer at 1732159468599Flushing aebdff1855164d839f0828203eeecd04/A: appending metadata at 1732159468603 (+4 ms)Flushing aebdff1855164d839f0828203eeecd04/A: closing flushed file at 1732159468603Flushing aebdff1855164d839f0828203eeecd04/B: creating writer at 1732159469015 (+412 ms)Flushing aebdff1855164d839f0828203eeecd04/B: appending metadata at 1732159469019 (+4 ms)Flushing aebdff1855164d839f0828203eeecd04/B: closing flushed file at 1732159469019Flushing aebdff1855164d839f0828203eeecd04/C: creating writer at 1732159469455 (+436 ms)Flushing aebdff1855164d839f0828203eeecd04/C: appending metadata at 1732159469458 (+3 ms)Flushing aebdff1855164d839f0828203eeecd04/C: closing flushed file at 1732159469458Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26b1502e: reopening flushed file at 1732159469870 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cd22ff: reopening flushed file at 1732159469877 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f46cb29: reopening flushed file at 1732159469884 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for aebdff1855164d839f0828203eeecd04 in 1294ms, sequenceid=496, compaction requested=true at 1732159469892 (+8 ms)Writing region close event to WAL at 1732159470030 (+138 ms)Running coprocessor post-close hooks at 1732159470034 (+4 ms)Closed at 1732159470034 2024-11-21T03:24:30,038 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:30,038 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=aebdff1855164d839f0828203eeecd04, regionState=CLOSED 2024-11-21T03:24:30,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:24:30,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-21T03:24:30,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure aebdff1855164d839f0828203eeecd04, server=b69e2a1ee061,45649,1732159441548 in 1.8170 sec 2024-11-21T03:24:30,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-21T03:24:30,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aebdff1855164d839f0828203eeecd04, UNASSIGN in 1.8230 sec 2024-11-21T03:24:30,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-21T03:24:30,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8300 sec 2024-11-21T03:24:30,048 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159470048"}]},"ts":"1732159470048"} 2024-11-21T03:24:30,050 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T03:24:30,050 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T03:24:30,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8450 sec 2024-11-21T03:24:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T03:24:30,358 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:30,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T03:24:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:30,368 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T03:24:30,370 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:30,375 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:30,380 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/recovered.edits] 2024-11-21T03:24:30,382 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/0be3b04399f84a6399415288bec103ec to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/0be3b04399f84a6399415288bec103ec 2024-11-21T03:24:30,383 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/a24ee87682d941c48f1e4e9e0b8d456f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/a24ee87682d941c48f1e4e9e0b8d456f 2024-11-21T03:24:30,384 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/dec0f70e77654f94a859edfd2e501bda to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/dec0f70e77654f94a859edfd2e501bda 2024-11-21T03:24:30,386 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e2059ea0d6214dac9c61897671267201 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e2059ea0d6214dac9c61897671267201 2024-11-21T03:24:30,387 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e544e01ab0964bceaf3c4b01f194ffb4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/A/e544e01ab0964bceaf3c4b01f194ffb4 2024-11-21T03:24:30,390 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/79935a64ea60419a804dbbb29e342628 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/79935a64ea60419a804dbbb29e342628 2024-11-21T03:24:30,391 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/8a5e98d1761c4f269648a214eb2ada78 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/8a5e98d1761c4f269648a214eb2ada78 2024-11-21T03:24:30,392 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/9ff41fde02f543a38d9c43816f87bcf9 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/9ff41fde02f543a38d9c43816f87bcf9 2024-11-21T03:24:30,394 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/a942fb0b06c74817baf37f95ed476fbc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/a942fb0b06c74817baf37f95ed476fbc 2024-11-21T03:24:30,395 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/ae814aed040045118168a042cdd67610 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/B/ae814aed040045118168a042cdd67610 2024-11-21T03:24:30,398 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/4c9b6d3a8a834fe59362b672a0e70d19 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/4c9b6d3a8a834fe59362b672a0e70d19 2024-11-21T03:24:30,399 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7f9b95570b73439eb8e9bfd0386a9350 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/7f9b95570b73439eb8e9bfd0386a9350 2024-11-21T03:24:30,401 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/81514e67f2e2451296025ebebbbb6bc6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/81514e67f2e2451296025ebebbbb6bc6 2024-11-21T03:24:30,402 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/c47f573254404d0891127e54599be4d5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/c47f573254404d0891127e54599be4d5 2024-11-21T03:24:30,403 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/f1bc63da66b24cdd93ee479b820f1c01 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/C/f1bc63da66b24cdd93ee479b820f1c01 2024-11-21T03:24:30,406 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/recovered.edits/499.seqid to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04/recovered.edits/499.seqid 2024-11-21T03:24:30,407 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/aebdff1855164d839f0828203eeecd04 2024-11-21T03:24:30,407 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T03:24:30,413 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:30,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-21T03:24:30,424 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T03:24:30,462 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T03:24:30,464 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:30,465 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T03:24:30,465 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732159470465"}]},"ts":"9223372036854775807"} 2024-11-21T03:24:30,471 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T03:24:30,471 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => aebdff1855164d839f0828203eeecd04, NAME => 'TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T03:24:30,471 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T03:24:30,471 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732159470471"}]},"ts":"9223372036854775807"} 2024-11-21T03:24:30,475 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T03:24:30,476 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T03:24:30,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 113 msec 2024-11-21T03:24:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T03:24:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T03:24:30,690 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:30,706 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_412655455_22 at /127.0.0.1:56256 [Waiting for operation #198] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_412655455_22 at /127.0.0.1:56004 [Waiting for operation #237] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1294884874_22 at /127.0.0.1:56152 [Waiting for operation #226] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;b69e2a1ee061:45649-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=860 (was 638) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2346 (was 3337) 2024-11-21T03:24:30,717 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=860, ProcessCount=11, AvailableMemoryMB=2346 2024-11-21T03:24:30,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:24:30,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:24:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:30,722 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:24:30,722 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:30,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-21T03:24:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T03:24:30,723 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:24:30,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741942_1118 (size=959) 2024-11-21T03:24:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T03:24:31,016 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T03:24:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T03:24:31,135 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1 2024-11-21T03:24:31,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741943_1119 (size=53) 2024-11-21T03:24:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T03:24:31,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:24:31,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 72b1610bcd54798702c9c5481e814b4c, disabling compactions & flushes 2024-11-21T03:24:31,545 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:31,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:31,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. after waiting 0 ms 2024-11-21T03:24:31,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:31,545 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:31,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 72b1610bcd54798702c9c5481e814b4c: Waiting for close lock at 1732159471545Disabling compacts and flushes for region at 1732159471545Disabling writes for close at 1732159471545Writing region close event to WAL at 1732159471545Closed at 1732159471545 2024-11-21T03:24:31,546 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:24:31,547 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732159471547"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732159471547"}]},"ts":"1732159471547"} 2024-11-21T03:24:31,549 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:24:31,550 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:24:31,550 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159471550"}]},"ts":"1732159471550"} 2024-11-21T03:24:31,552 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:24:31,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, ASSIGN}] 2024-11-21T03:24:31,553 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, ASSIGN 2024-11-21T03:24:31,554 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, ASSIGN; state=OFFLINE, location=b69e2a1ee061,45649,1732159441548; forceNewPlan=false, retain=false 2024-11-21T03:24:31,705 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=72b1610bcd54798702c9c5481e814b4c, regionState=OPENING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:31,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, ASSIGN because future has completed 2024-11-21T03:24:31,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:24:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T03:24:31,885 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:31,886 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:24:31,886 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,887 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:24:31,887 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,887 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,889 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,892 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72b1610bcd54798702c9c5481e814b4c columnFamilyName A 2024-11-21T03:24:31,892 DEBUG [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:31,892 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(327): Store=72b1610bcd54798702c9c5481e814b4c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:31,893 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,894 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72b1610bcd54798702c9c5481e814b4c columnFamilyName B 2024-11-21T03:24:31,894 DEBUG [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:31,894 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(327): Store=72b1610bcd54798702c9c5481e814b4c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:31,894 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,895 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72b1610bcd54798702c9c5481e814b4c columnFamilyName C 2024-11-21T03:24:31,895 DEBUG [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:31,896 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(327): Store=72b1610bcd54798702c9c5481e814b4c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:31,896 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,896 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,897 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,898 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,898 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,898 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:24:31,899 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,901 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:24:31,901 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 72b1610bcd54798702c9c5481e814b4c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63588390, jitterRate=-0.052459150552749634}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:24:31,901 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:31,902 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 72b1610bcd54798702c9c5481e814b4c: Running coprocessor pre-open hook at 1732159471888Writing region info on filesystem at 1732159471888Initializing all the Stores at 1732159471889 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159471889Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159471889Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159471889Cleaning up temporary data from old regions at 1732159471898 (+9 ms)Running coprocessor post-open hooks at 1732159471901 (+3 ms)Region opened successfully at 1732159471902 (+1 ms) 2024-11-21T03:24:31,903 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., pid=34, masterSystemTime=1732159471879 2024-11-21T03:24:31,904 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:31,904 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:31,905 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=72b1610bcd54798702c9c5481e814b4c, regionState=OPEN, openSeqNum=2, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:31,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:24:31,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-21T03:24:31,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 in 191 msec 2024-11-21T03:24:31,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-21T03:24:31,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, ASSIGN in 357 msec 2024-11-21T03:24:31,911 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:24:31,911 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159471911"}]},"ts":"1732159471911"} 2024-11-21T03:24:31,913 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:24:31,914 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:24:31,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1940 sec 2024-11-21T03:24:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T03:24:32,878 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:32,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:32,879 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:32,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:32,880 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6090732d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:32,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:32,883 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f289009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:32,884 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:32,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:32,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:32,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:32,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:24:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is b69e2a1ee061,42023,1732159440464 2024-11-21T03:24:32,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@281d24f1 2024-11-21T03:24:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:24:32,890 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:24:32,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:24:32,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:24:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:32,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741944_1120 (size=995) 2024-11-21T03:24:33,315 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T03:24:33,315 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T03:24:33,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:24:33,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, REOPEN/MOVE}] 2024-11-21T03:24:33,330 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, REOPEN/MOVE 2024-11-21T03:24:33,331 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=72b1610bcd54798702c9c5481e814b4c, regionState=CLOSING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:33,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, REOPEN/MOVE because future has completed 2024-11-21T03:24:33,336 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:24:33,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:24:33,489 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,489 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:24:33,490 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 72b1610bcd54798702c9c5481e814b4c, disabling compactions & flushes 2024-11-21T03:24:33,490 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:33,490 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:33,490 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. after waiting 0 ms 2024-11-21T03:24:33,490 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:33,494 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T03:24:33,495 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:33,495 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 72b1610bcd54798702c9c5481e814b4c: Waiting for close lock at 1732159473489Running coprocessor pre-close hooks at 1732159473489Disabling compacts and flushes for region at 1732159473489Disabling writes for close at 1732159473490 (+1 ms)Writing region close event to WAL at 1732159473490Running coprocessor post-close hooks at 1732159473495 (+5 ms)Closed at 1732159473495 2024-11-21T03:24:33,495 WARN [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 72b1610bcd54798702c9c5481e814b4c to self. 2024-11-21T03:24:33,498 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=72b1610bcd54798702c9c5481e814b4c, regionState=CLOSED 2024-11-21T03:24:33,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:24:33,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-21T03:24:33,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 in 168 msec 2024-11-21T03:24:33,507 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, REOPEN/MOVE; state=CLOSED, location=b69e2a1ee061,45649,1732159441548; forceNewPlan=false, retain=true 2024-11-21T03:24:33,658 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=72b1610bcd54798702c9c5481e814b4c, regionState=OPENING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:33,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, REOPEN/MOVE because future has completed 2024-11-21T03:24:33,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:24:33,820 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:33,821 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:24:33,821 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,821 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:24:33,821 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,821 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,826 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,831 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72b1610bcd54798702c9c5481e814b4c columnFamilyName A 2024-11-21T03:24:33,834 DEBUG [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:33,834 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(327): Store=72b1610bcd54798702c9c5481e814b4c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:33,835 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,835 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72b1610bcd54798702c9c5481e814b4c columnFamilyName B 2024-11-21T03:24:33,836 DEBUG [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:33,836 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(327): Store=72b1610bcd54798702c9c5481e814b4c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:33,836 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,837 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72b1610bcd54798702c9c5481e814b4c columnFamilyName C 2024-11-21T03:24:33,837 DEBUG [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:33,838 INFO [StoreOpener-72b1610bcd54798702c9c5481e814b4c-1 {}] regionserver.HStore(327): Store=72b1610bcd54798702c9c5481e814b4c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:24:33,838 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,839 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,841 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,842 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,842 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,842 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:24:33,844 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,846 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 72b1610bcd54798702c9c5481e814b4c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65468818, jitterRate=-0.024438589811325073}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:24:33,846 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,848 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 72b1610bcd54798702c9c5481e814b4c: Running coprocessor pre-open hook at 1732159473821Writing region info on filesystem at 1732159473821Initializing all the Stores at 1732159473822 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159473822Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159473822Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159473822Cleaning up temporary data from old regions at 1732159473842 (+20 ms)Running coprocessor post-open hooks at 1732159473846 (+4 ms)Region opened successfully at 1732159473848 (+2 ms) 2024-11-21T03:24:33,849 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., pid=39, masterSystemTime=1732159473817 2024-11-21T03:24:33,852 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:33,852 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:33,853 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=72b1610bcd54798702c9c5481e814b4c, regionState=OPEN, openSeqNum=5, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:33,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:24:33,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-21T03:24:33,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 in 194 msec 2024-11-21T03:24:33,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-21T03:24:33,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, REOPEN/MOVE in 533 msec 2024-11-21T03:24:33,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-21T03:24:33,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 553 msec 2024-11-21T03:24:33,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 981 msec 2024-11-21T03:24:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-21T03:24:33,882 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:33,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:33,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:33,885 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35807dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:33,887 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee4e7d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:33,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:33,898 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4feb4bcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:33,900 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8faf53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:33,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:33,904 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c036b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:33,907 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183bc2ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:33,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:33,910 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65c207d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:33,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e18015f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:33,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:33,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24f7224c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:33,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297218fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:33,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:33,927 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:33,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:33,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c85dbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:33,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:33,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:33,930 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b86a1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:33,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:33,933 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa9aa55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:33,935 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3eeec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:33,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:33,938 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25607401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:33,940 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5784fa65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:24:33,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:24:33,943 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:24:33,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:24:33,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:24:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9fb7d5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:24:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:24:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:33,946 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:24:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2149b6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:24:33,947 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:33,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:24:33,972 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:33,973 DEBUG [Thread-619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:33,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:33,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:24:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:24:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:33,984 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:33,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:24:33,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-21T03:24:33,989 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:24:33,991 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:33,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:33,996 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:33,997 DEBUG [Thread-622 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:33,998 DEBUG [Thread-622 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:33,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:34,001 DEBUG [Thread-622 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:24:34,008 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:34,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:24:34,017 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:34,019 DEBUG [Thread-611 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:34,020 DEBUG [Thread-611 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:34,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:24:34,033 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:34,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:34,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:24:34,041 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:34,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:34,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:34,059 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:24:34,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:24:34,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159534047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159534058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:24:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:24:34,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:24:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:24:34,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159534059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159534061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159534063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:24:34,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112176c59b59533448f6a7bad5739886e22c_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_1/A:col10/1732159473980/Put/seqid=0 2024-11-21T03:24:34,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741945_1121 (size=14594) 2024-11-21T03:24:34,130 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:34,137 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112176c59b59533448f6a7bad5739886e22c_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112176c59b59533448f6a7bad5739886e22c_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:34,141 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/5d7779b05dd5414d86bcb2a91f84f5ff, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:34,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:34,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/5d7779b05dd5414d86bcb2a91f84f5ff is 175, key is test_row_1/A:col10/1732159473980/Put/seqid=0 2024-11-21T03:24:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741946_1122 (size=39545) 2024-11-21T03:24:34,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159534173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159534176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159534179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159534179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159534180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:34,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:34,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:24:34,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159534387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159534388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159534398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159534398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159534400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/5d7779b05dd5414d86bcb2a91f84f5ff 2024-11-21T03:24:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:24:34,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/6f47862db40344359592c23afee79ab9 is 50, key is test_row_1/B:col10/1732159473980/Put/seqid=0 2024-11-21T03:24:34,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741947_1123 (size=9657) 2024-11-21T03:24:34,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/6f47862db40344359592c23afee79ab9 2024-11-21T03:24:34,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/606f32f6ed2d4904ad8a987f0dc2201f is 50, key is test_row_1/C:col10/1732159473980/Put/seqid=0 2024-11-21T03:24:34,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741948_1124 (size=9657) 2024-11-21T03:24:34,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159534697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159534697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159534708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159534708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159534717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:34,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:34,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:34,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:34,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:34,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:34,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/606f32f6ed2d4904ad8a987f0dc2201f 2024-11-21T03:24:35,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:35,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:35,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/5d7779b05dd5414d86bcb2a91f84f5ff as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/5d7779b05dd5414d86bcb2a91f84f5ff 2024-11-21T03:24:35,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/5d7779b05dd5414d86bcb2a91f84f5ff, entries=200, sequenceid=15, filesize=38.6 K 2024-11-21T03:24:35,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/6f47862db40344359592c23afee79ab9 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/6f47862db40344359592c23afee79ab9 2024-11-21T03:24:35,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/6f47862db40344359592c23afee79ab9, entries=100, sequenceid=15, filesize=9.4 K 2024-11-21T03:24:35,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/606f32f6ed2d4904ad8a987f0dc2201f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/606f32f6ed2d4904ad8a987f0dc2201f 2024-11-21T03:24:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:24:35,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/606f32f6ed2d4904ad8a987f0dc2201f, entries=100, sequenceid=15, filesize=9.4 K 2024-11-21T03:24:35,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 72b1610bcd54798702c9c5481e814b4c in 1165ms, sequenceid=15, compaction requested=false 2024-11-21T03:24:35,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:35,216 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:35,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T03:24:35,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219894397ea69c4bd5a56a461d5785f33d_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159475210/Put/seqid=0 2024-11-21T03:24:35,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:35,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:35,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159535241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159535242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159535242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159535247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159535244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741949_1125 (size=14594) 2024-11-21T03:24:35,262 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,268 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219894397ea69c4bd5a56a461d5785f33d_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219894397ea69c4bd5a56a461d5785f33d_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:35,270 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/8ac66fb50a874f9ea3f31c8be864223e, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:35,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/8ac66fb50a874f9ea3f31c8be864223e is 175, key is test_row_0/A:col10/1732159475210/Put/seqid=0 2024-11-21T03:24:35,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741950_1126 (size=39549) 2024-11-21T03:24:35,315 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/8ac66fb50a874f9ea3f31c8be864223e 2024-11-21T03:24:35,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/d2d9f52c8a2f43b2bf2ee45ff8e0ef2b is 50, key is test_row_0/B:col10/1732159475210/Put/seqid=0 2024-11-21T03:24:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159535369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159535370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741951_1127 (size=12001) 2024-11-21T03:24:35,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159535377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/d2d9f52c8a2f43b2bf2ee45ff8e0ef2b 2024-11-21T03:24:35,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159535376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159535377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:35,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:35,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/4946f24ce0894c12aa5a7c82b7266574 is 50, key is test_row_0/C:col10/1732159475210/Put/seqid=0 2024-11-21T03:24:35,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741952_1128 (size=12001) 2024-11-21T03:24:35,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:35,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:35,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159535578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159535578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,582 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:24:35,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159535588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159535588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159535589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:35,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:35,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:35,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:35,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:35,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:35,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/4946f24ce0894c12aa5a7c82b7266574 2024-11-21T03:24:35,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/8ac66fb50a874f9ea3f31c8be864223e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8ac66fb50a874f9ea3f31c8be864223e 2024-11-21T03:24:35,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159535888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159535888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159535898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159535899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159535899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:35,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:35,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8ac66fb50a874f9ea3f31c8be864223e, entries=200, sequenceid=42, filesize=38.6 K 2024-11-21T03:24:35,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/d2d9f52c8a2f43b2bf2ee45ff8e0ef2b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d2d9f52c8a2f43b2bf2ee45ff8e0ef2b 2024-11-21T03:24:35,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d2d9f52c8a2f43b2bf2ee45ff8e0ef2b, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T03:24:35,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/4946f24ce0894c12aa5a7c82b7266574 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/4946f24ce0894c12aa5a7c82b7266574 2024-11-21T03:24:35,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/4946f24ce0894c12aa5a7c82b7266574, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T03:24:35,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 72b1610bcd54798702c9c5481e814b4c in 715ms, sequenceid=42, compaction requested=false 2024-11-21T03:24:35,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:36,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T03:24:36,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:36,049 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-21T03:24:36,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212fb66512d14f41eb952c55881e9f24d3_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159475243/Put/seqid=0 2024-11-21T03:24:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:24:36,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741953_1129 (size=12154) 2024-11-21T03:24:36,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:36,183 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212fb66512d14f41eb952c55881e9f24d3_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212fb66512d14f41eb952c55881e9f24d3_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:36,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/aa2c73b3570840d394507c3488bb9eac, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:36,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/aa2c73b3570840d394507c3488bb9eac is 175, key is test_row_0/A:col10/1732159475243/Put/seqid=0 2024-11-21T03:24:36,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741954_1130 (size=30955) 2024-11-21T03:24:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:36,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:36,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159536482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159536488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159536489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159536497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159536497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159536608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159536609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159536612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159536618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159536618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/aa2c73b3570840d394507c3488bb9eac 2024-11-21T03:24:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/7dc7a4cc19304aaaae08725c32aa2e18 is 50, key is test_row_0/B:col10/1732159475243/Put/seqid=0 2024-11-21T03:24:36,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741955_1131 (size=12001) 2024-11-21T03:24:36,725 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/7dc7a4cc19304aaaae08725c32aa2e18 2024-11-21T03:24:36,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/f18b7573e3884533ad7fa735fd3dc2f0 is 50, key is test_row_0/C:col10/1732159475243/Put/seqid=0 2024-11-21T03:24:36,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741956_1132 (size=12001) 2024-11-21T03:24:36,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159536819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159536819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,824 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/f18b7573e3884533ad7fa735fd3dc2f0 2024-11-21T03:24:36,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159536828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159536837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159536839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:36,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:36,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/aa2c73b3570840d394507c3488bb9eac as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/aa2c73b3570840d394507c3488bb9eac 2024-11-21T03:24:36,857 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/aa2c73b3570840d394507c3488bb9eac, entries=150, sequenceid=51, filesize=30.2 K 2024-11-21T03:24:36,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/7dc7a4cc19304aaaae08725c32aa2e18 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7dc7a4cc19304aaaae08725c32aa2e18 2024-11-21T03:24:36,868 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7dc7a4cc19304aaaae08725c32aa2e18, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T03:24:36,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/f18b7573e3884533ad7fa735fd3dc2f0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/f18b7573e3884533ad7fa735fd3dc2f0 2024-11-21T03:24:36,885 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/f18b7573e3884533ad7fa735fd3dc2f0, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T03:24:36,889 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 72b1610bcd54798702c9c5481e814b4c in 840ms, sequenceid=51, compaction requested=true 2024-11-21T03:24:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:36,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-21T03:24:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-21T03:24:36,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-21T03:24:36,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9010 sec 2024-11-21T03:24:36,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.9110 sec 2024-11-21T03:24:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:37,144 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:37,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-21T03:24:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159537166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159537168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159537168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159537169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159537183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212082bd3550744f3095729924138e8b3b_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159476492/Put/seqid=0 2024-11-21T03:24:37,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741957_1133 (size=12154) 2024-11-21T03:24:37,239 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:37,262 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212082bd3550744f3095729924138e8b3b_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212082bd3550744f3095729924138e8b3b_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:37,265 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/649a98816e5d412c945072ea960fc695, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:37,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/649a98816e5d412c945072ea960fc695 is 175, key is test_row_0/A:col10/1732159476492/Put/seqid=0 2024-11-21T03:24:37,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741958_1134 (size=30955) 2024-11-21T03:24:37,282 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/649a98816e5d412c945072ea960fc695 2024-11-21T03:24:37,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159537287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159537291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159537291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159537291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159537298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/f023cd0d553645648da2d7e17530bffc is 50, key is test_row_0/B:col10/1732159476492/Put/seqid=0 2024-11-21T03:24:37,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741959_1135 (size=12001) 2024-11-21T03:24:37,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/f023cd0d553645648da2d7e17530bffc 2024-11-21T03:24:37,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159537501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/cabb5fe518474b5d9840019994d9d6c2 is 50, key is test_row_0/C:col10/1732159476492/Put/seqid=0 2024-11-21T03:24:37,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159537508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159537508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159537512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159537513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741960_1136 (size=12001) 2024-11-21T03:24:37,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/cabb5fe518474b5d9840019994d9d6c2 2024-11-21T03:24:37,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/649a98816e5d412c945072ea960fc695 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/649a98816e5d412c945072ea960fc695 2024-11-21T03:24:37,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/649a98816e5d412c945072ea960fc695, entries=150, sequenceid=80, filesize=30.2 K 2024-11-21T03:24:37,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/f023cd0d553645648da2d7e17530bffc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f023cd0d553645648da2d7e17530bffc 2024-11-21T03:24:37,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f023cd0d553645648da2d7e17530bffc, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T03:24:37,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/cabb5fe518474b5d9840019994d9d6c2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/cabb5fe518474b5d9840019994d9d6c2 2024-11-21T03:24:37,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/cabb5fe518474b5d9840019994d9d6c2, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T03:24:37,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 72b1610bcd54798702c9c5481e814b4c in 521ms, sequenceid=80, compaction requested=true 2024-11-21T03:24:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:24:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:37,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T03:24:37,665 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:37,667 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:37,667 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/A is initiating minor compaction (all files) 2024-11-21T03:24:37,667 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/A in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:37,668 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/5d7779b05dd5414d86bcb2a91f84f5ff, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8ac66fb50a874f9ea3f31c8be864223e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/aa2c73b3570840d394507c3488bb9eac, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/649a98816e5d412c945072ea960fc695] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=137.7 K 2024-11-21T03:24:37,668 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:37,668 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:37,668 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/5d7779b05dd5414d86bcb2a91f84f5ff, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8ac66fb50a874f9ea3f31c8be864223e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/aa2c73b3570840d394507c3488bb9eac, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/649a98816e5d412c945072ea960fc695] 2024-11-21T03:24:37,672 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d7779b05dd5414d86bcb2a91f84f5ff, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732159473971 2024-11-21T03:24:37,677 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ac66fb50a874f9ea3f31c8be864223e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732159474041 2024-11-21T03:24:37,677 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:37,677 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa2c73b3570840d394507c3488bb9eac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159475235 2024-11-21T03:24:37,677 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/C is initiating minor compaction (all files) 2024-11-21T03:24:37,677 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/C in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:37,677 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/606f32f6ed2d4904ad8a987f0dc2201f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/4946f24ce0894c12aa5a7c82b7266574, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/f18b7573e3884533ad7fa735fd3dc2f0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/cabb5fe518474b5d9840019994d9d6c2] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=44.6 K 2024-11-21T03:24:37,678 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 649a98816e5d412c945072ea960fc695, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732159476485 2024-11-21T03:24:37,678 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 606f32f6ed2d4904ad8a987f0dc2201f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732159473980 2024-11-21T03:24:37,678 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4946f24ce0894c12aa5a7c82b7266574, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732159474052 2024-11-21T03:24:37,679 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting f18b7573e3884533ad7fa735fd3dc2f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159475235 2024-11-21T03:24:37,679 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting cabb5fe518474b5d9840019994d9d6c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732159476485 2024-11-21T03:24:37,699 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:37,705 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121662fe28bfd5344cd866e38e0d7aba38d_72b1610bcd54798702c9c5481e814b4c store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:37,715 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121662fe28bfd5344cd866e38e0d7aba38d_72b1610bcd54798702c9c5481e814b4c, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:37,715 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121662fe28bfd5344cd866e38e0d7aba38d_72b1610bcd54798702c9c5481e814b4c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:37,718 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#C#compaction#118 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:37,719 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/8ef92fbf3f904e49a55bba85db7ee987 is 50, key is test_row_0/C:col10/1732159476492/Put/seqid=0 2024-11-21T03:24:37,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741961_1137 (size=4469) 2024-11-21T03:24:37,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741962_1138 (size=12139) 2024-11-21T03:24:37,773 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/8ef92fbf3f904e49a55bba85db7ee987 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8ef92fbf3f904e49a55bba85db7ee987 2024-11-21T03:24:37,787 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/C of 72b1610bcd54798702c9c5481e814b4c into 8ef92fbf3f904e49a55bba85db7ee987(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:37,787 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:37,787 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/C, priority=12, startTime=1732159477665; duration=0sec 2024-11-21T03:24:37,788 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:37,788 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:C 2024-11-21T03:24:37,789 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:37,790 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:37,791 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/B is initiating minor compaction (all files) 2024-11-21T03:24:37,791 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/B in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:37,791 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/6f47862db40344359592c23afee79ab9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d2d9f52c8a2f43b2bf2ee45ff8e0ef2b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7dc7a4cc19304aaaae08725c32aa2e18, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f023cd0d553645648da2d7e17530bffc] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=44.6 K 2024-11-21T03:24:37,792 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f47862db40344359592c23afee79ab9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732159473980 2024-11-21T03:24:37,792 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d2d9f52c8a2f43b2bf2ee45ff8e0ef2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732159474052 2024-11-21T03:24:37,792 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dc7a4cc19304aaaae08725c32aa2e18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159475235 2024-11-21T03:24:37,793 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting f023cd0d553645648da2d7e17530bffc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732159476485 2024-11-21T03:24:37,841 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:37,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:24:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:37,867 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#B#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:37,867 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/5ed5f65a43e4475092f703108cdb897b is 50, key is test_row_0/B:col10/1732159476492/Put/seqid=0 2024-11-21T03:24:37,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212cb211c9bd76486c867d8367311dec6c_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159477157/Put/seqid=0 2024-11-21T03:24:37,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741963_1139 (size=12139) 2024-11-21T03:24:37,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159537924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159537927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159537929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159537930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159537930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:37,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741964_1140 (size=12154) 2024-11-21T03:24:37,973 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:38,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212cb211c9bd76486c867d8367311dec6c_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212cb211c9bd76486c867d8367311dec6c_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:38,018 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/b54067b87d7d4a69ae4ed3e01f33fed2, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:38,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/b54067b87d7d4a69ae4ed3e01f33fed2 is 175, key is test_row_0/A:col10/1732159477157/Put/seqid=0 2024-11-21T03:24:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159538048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159538048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159538048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159538049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741965_1141 (size=30955) 2024-11-21T03:24:38,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159538058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,140 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#A#compaction#117 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:38,141 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/df3e6e29252c476987ed0bdbb238d473 is 175, key is test_row_0/A:col10/1732159476492/Put/seqid=0 2024-11-21T03:24:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:24:38,158 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:38,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-21T03:24:38,166 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T03:24:38,170 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:38,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:38,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741966_1142 (size=31093) 2024-11-21T03:24:38,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159538268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159538269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159538269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159538269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T03:24:38,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159538279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T03:24:38,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:38,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:38,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:38,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:38,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:38,338 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/5ed5f65a43e4475092f703108cdb897b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/5ed5f65a43e4475092f703108cdb897b 2024-11-21T03:24:38,356 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/B of 72b1610bcd54798702c9c5481e814b4c into 5ed5f65a43e4475092f703108cdb897b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:38,357 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:38,357 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/B, priority=12, startTime=1732159477665; duration=0sec 2024-11-21T03:24:38,357 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:38,357 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:B 2024-11-21T03:24:38,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/b54067b87d7d4a69ae4ed3e01f33fed2 2024-11-21T03:24:38,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T03:24:38,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:38,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:38,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/940e69c397dc4c15ac58d68537a8a81d is 50, key is test_row_0/B:col10/1732159477157/Put/seqid=0 2024-11-21T03:24:38,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T03:24:38,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741967_1143 (size=12001) 2024-11-21T03:24:38,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/940e69c397dc4c15ac58d68537a8a81d 2024-11-21T03:24:38,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/c9d32a2441994a81b12901d0b40c9c10 is 50, key is test_row_0/C:col10/1732159477157/Put/seqid=0 2024-11-21T03:24:38,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159538581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159538581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159538581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159538589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159538602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:38,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:38,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:38,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741968_1144 (size=12001) 2024-11-21T03:24:38,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/c9d32a2441994a81b12901d0b40c9c10 2024-11-21T03:24:38,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T03:24:38,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:38,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:38,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:38,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:38,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:38,647 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/df3e6e29252c476987ed0bdbb238d473 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/df3e6e29252c476987ed0bdbb238d473 2024-11-21T03:24:38,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/b54067b87d7d4a69ae4ed3e01f33fed2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b54067b87d7d4a69ae4ed3e01f33fed2 2024-11-21T03:24:38,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b54067b87d7d4a69ae4ed3e01f33fed2, entries=150, sequenceid=94, filesize=30.2 K 2024-11-21T03:24:38,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/940e69c397dc4c15ac58d68537a8a81d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/940e69c397dc4c15ac58d68537a8a81d 2024-11-21T03:24:38,667 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/A of 72b1610bcd54798702c9c5481e814b4c into df3e6e29252c476987ed0bdbb238d473(size=30.4 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-21T03:24:38,667 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:38,667 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/A, priority=12, startTime=1732159477665; duration=1sec 2024-11-21T03:24:38,667 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:38,667 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:A 2024-11-21T03:24:38,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/940e69c397dc4c15ac58d68537a8a81d, entries=150, sequenceid=94, filesize=11.7 K 2024-11-21T03:24:38,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/c9d32a2441994a81b12901d0b40c9c10 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c9d32a2441994a81b12901d0b40c9c10 2024-11-21T03:24:38,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c9d32a2441994a81b12901d0b40c9c10, entries=150, sequenceid=94, filesize=11.7 K 2024-11-21T03:24:38,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 72b1610bcd54798702c9c5481e814b4c in 854ms, sequenceid=94, compaction requested=false 2024-11-21T03:24:38,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:38,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T03:24:38,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:38,804 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:24:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T03:24:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112168561e3bd6aa4619bba3886e53f0f208_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159477923/Put/seqid=0 2024-11-21T03:24:38,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741969_1145 (size=12154) 2024-11-21T03:24:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:39,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159539141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159539141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159539145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159539145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159539139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:39,259 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112168561e3bd6aa4619bba3886e53f0f208_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112168561e3bd6aa4619bba3886e53f0f208_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:39,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/04615b0b1cad4b1487a497c84fb90ff0, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:39,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/04615b0b1cad4b1487a497c84fb90ff0 is 175, key is test_row_0/A:col10/1732159477923/Put/seqid=0 2024-11-21T03:24:39,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159539262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159539262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159539262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159539264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159539264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741970_1146 (size=30955) 2024-11-21T03:24:39,310 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/04615b0b1cad4b1487a497c84fb90ff0 2024-11-21T03:24:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T03:24:39,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/4424cdf1ec2d41c794bccda89bb6adfa is 50, key is test_row_0/B:col10/1732159477923/Put/seqid=0 2024-11-21T03:24:39,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741971_1147 (size=12001) 2024-11-21T03:24:39,375 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/4424cdf1ec2d41c794bccda89bb6adfa 2024-11-21T03:24:39,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/10309e96d80f4155ae3f9ce431753b03 is 50, key is test_row_0/C:col10/1732159477923/Put/seqid=0 2024-11-21T03:24:39,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741972_1148 (size=12001) 2024-11-21T03:24:39,439 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/10309e96d80f4155ae3f9ce431753b03 2024-11-21T03:24:39,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/04615b0b1cad4b1487a497c84fb90ff0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/04615b0b1cad4b1487a497c84fb90ff0 2024-11-21T03:24:39,475 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/04615b0b1cad4b1487a497c84fb90ff0, entries=150, sequenceid=119, filesize=30.2 K 2024-11-21T03:24:39,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/4424cdf1ec2d41c794bccda89bb6adfa as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4424cdf1ec2d41c794bccda89bb6adfa 2024-11-21T03:24:39,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159539479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,496 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4424cdf1ec2d41c794bccda89bb6adfa, entries=150, sequenceid=119, filesize=11.7 K 2024-11-21T03:24:39,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/10309e96d80f4155ae3f9ce431753b03 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/10309e96d80f4155ae3f9ce431753b03 2024-11-21T03:24:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159539480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159539488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159539498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,507 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/10309e96d80f4155ae3f9ce431753b03, entries=150, sequenceid=119, filesize=11.7 K 2024-11-21T03:24:39,513 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 72b1610bcd54798702c9c5481e814b4c in 708ms, sequenceid=119, compaction requested=true 2024-11-21T03:24:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:39,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-21T03:24:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-21T03:24:39,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-21T03:24:39,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3450 sec 2024-11-21T03:24:39,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.3620 sec 2024-11-21T03:24:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:39,534 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:39,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:24:39,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121da1a08f942f04b9c92c6da03ae3af7c1_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159479133/Put/seqid=0 2024-11-21T03:24:39,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741973_1149 (size=14794) 2024-11-21T03:24:39,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159539754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159539798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159539807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159539807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159539808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:39,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159539867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:39,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,043 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121da1a08f942f04b9c92c6da03ae3af7c1_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121da1a08f942f04b9c92c6da03ae3af7c1_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:40,051 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/ba15576c3e1349e1b68fc5064dbff875, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:40,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/ba15576c3e1349e1b68fc5064dbff875 is 175, key is test_row_0/A:col10/1732159479133/Put/seqid=0 2024-11-21T03:24:40,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741974_1150 (size=39749) 2024-11-21T03:24:40,073 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/ba15576c3e1349e1b68fc5064dbff875 2024-11-21T03:24:40,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159540082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/899829f77bc546e093c6232ae05dd6fd is 50, key is test_row_0/B:col10/1732159479133/Put/seqid=0 2024-11-21T03:24:40,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741975_1151 (size=12151) 2024-11-21T03:24:40,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/899829f77bc546e093c6232ae05dd6fd 2024-11-21T03:24:40,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/76dd3583748242f8974b94f560593bf6 is 50, key is test_row_0/C:col10/1732159479133/Put/seqid=0 2024-11-21T03:24:40,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741976_1152 (size=12151) 2024-11-21T03:24:40,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/76dd3583748242f8974b94f560593bf6 2024-11-21T03:24:40,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/ba15576c3e1349e1b68fc5064dbff875 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/ba15576c3e1349e1b68fc5064dbff875 2024-11-21T03:24:40,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/ba15576c3e1349e1b68fc5064dbff875, entries=200, sequenceid=133, filesize=38.8 K 2024-11-21T03:24:40,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/899829f77bc546e093c6232ae05dd6fd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/899829f77bc546e093c6232ae05dd6fd 2024-11-21T03:24:40,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/899829f77bc546e093c6232ae05dd6fd, entries=150, sequenceid=133, filesize=11.9 K 2024-11-21T03:24:40,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/76dd3583748242f8974b94f560593bf6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/76dd3583748242f8974b94f560593bf6 2024-11-21T03:24:40,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/76dd3583748242f8974b94f560593bf6, entries=150, sequenceid=133, filesize=11.9 K 2024-11-21T03:24:40,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 72b1610bcd54798702c9c5481e814b4c in 682ms, sequenceid=133, compaction requested=true 2024-11-21T03:24:40,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:40,217 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:40,217 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:40,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:40,218 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:40,218 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132752 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:40,219 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/B is initiating minor compaction (all files) 2024-11-21T03:24:40,219 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/A is initiating minor compaction (all files) 2024-11-21T03:24:40,219 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/B in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,219 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/A in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,219 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/df3e6e29252c476987ed0bdbb238d473, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b54067b87d7d4a69ae4ed3e01f33fed2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/04615b0b1cad4b1487a497c84fb90ff0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/ba15576c3e1349e1b68fc5064dbff875] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=129.6 K 2024-11-21T03:24:40,219 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/5ed5f65a43e4475092f703108cdb897b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/940e69c397dc4c15ac58d68537a8a81d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4424cdf1ec2d41c794bccda89bb6adfa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/899829f77bc546e093c6232ae05dd6fd] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=47.2 K 2024-11-21T03:24:40,219 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,219 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/df3e6e29252c476987ed0bdbb238d473, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b54067b87d7d4a69ae4ed3e01f33fed2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/04615b0b1cad4b1487a497c84fb90ff0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/ba15576c3e1349e1b68fc5064dbff875] 2024-11-21T03:24:40,219 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed5f65a43e4475092f703108cdb897b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732159476485 2024-11-21T03:24:40,220 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting df3e6e29252c476987ed0bdbb238d473, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732159476485 2024-11-21T03:24:40,220 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 940e69c397dc4c15ac58d68537a8a81d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732159477157 2024-11-21T03:24:40,221 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b54067b87d7d4a69ae4ed3e01f33fed2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732159477157 2024-11-21T03:24:40,221 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4424cdf1ec2d41c794bccda89bb6adfa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732159477904 2024-11-21T03:24:40,221 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04615b0b1cad4b1487a497c84fb90ff0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732159477904 2024-11-21T03:24:40,221 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 899829f77bc546e093c6232ae05dd6fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159479133 2024-11-21T03:24:40,222 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba15576c3e1349e1b68fc5064dbff875, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159479133 2024-11-21T03:24:40,243 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:40,245 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:40,245 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/b30d17dc243840d3947e14be38a7c104 is 50, key is test_row_0/B:col10/1732159479133/Put/seqid=0 2024-11-21T03:24:40,246 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211303574083fc43aea5d22fe94075df84_72b1610bcd54798702c9c5481e814b4c store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:40,250 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211303574083fc43aea5d22fe94075df84_72b1610bcd54798702c9c5481e814b4c, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:40,250 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211303574083fc43aea5d22fe94075df84_72b1610bcd54798702c9c5481e814b4c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741977_1153 (size=12425) 2024-11-21T03:24:40,271 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/b30d17dc243840d3947e14be38a7c104 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b30d17dc243840d3947e14be38a7c104 2024-11-21T03:24:40,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741978_1154 (size=4469) 2024-11-21T03:24:40,278 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#A#compaction#130 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:40,278 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/8777862263f44cb5acd30211cf5acd11 is 175, key is test_row_0/A:col10/1732159479133/Put/seqid=0 2024-11-21T03:24:40,283 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/B of 72b1610bcd54798702c9c5481e814b4c into b30d17dc243840d3947e14be38a7c104(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:40,283 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:40,283 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/B, priority=12, startTime=1732159480217; duration=0sec 2024-11-21T03:24:40,283 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:40,283 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:B 2024-11-21T03:24:40,283 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:24:40,285 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:24:40,285 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/C is initiating minor compaction (all files) 2024-11-21T03:24:40,285 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/C in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,286 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8ef92fbf3f904e49a55bba85db7ee987, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c9d32a2441994a81b12901d0b40c9c10, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/10309e96d80f4155ae3f9ce431753b03, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/76dd3583748242f8974b94f560593bf6] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=47.2 K 2024-11-21T03:24:40,286 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ef92fbf3f904e49a55bba85db7ee987, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732159476485 2024-11-21T03:24:40,288 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting c9d32a2441994a81b12901d0b40c9c10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732159477157 2024-11-21T03:24:40,289 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 10309e96d80f4155ae3f9ce431753b03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732159477904 2024-11-21T03:24:40,290 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 76dd3583748242f8974b94f560593bf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159479133 2024-11-21T03:24:40,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741979_1155 (size=31379) 2024-11-21T03:24:40,316 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/8777862263f44cb5acd30211cf5acd11 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8777862263f44cb5acd30211cf5acd11 2024-11-21T03:24:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:40,322 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:40,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:24:40,323 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/A of 72b1610bcd54798702c9c5481e814b4c into 8777862263f44cb5acd30211cf5acd11(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:40,323 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:40,323 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/A, priority=12, startTime=1732159480216; duration=0sec 2024-11-21T03:24:40,323 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:40,323 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:A 2024-11-21T03:24:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T03:24:40,339 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:40,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-21T03:24:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T03:24:40,343 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:40,343 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:40,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:40,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159540339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159540341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,347 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#C#compaction#131 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:40,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159540343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,353 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/c3cb81ec529341b2bb17682f4482e3bc is 50, key is test_row_0/C:col10/1732159479133/Put/seqid=0 2024-11-21T03:24:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a492a031c7624bc28dbf198f05f527ed_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159479744/Put/seqid=0 2024-11-21T03:24:40,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159540344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741980_1156 (size=12425) 2024-11-21T03:24:40,403 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/c3cb81ec529341b2bb17682f4482e3bc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c3cb81ec529341b2bb17682f4482e3bc 2024-11-21T03:24:40,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159540398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741981_1157 (size=17284) 2024-11-21T03:24:40,409 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,414 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/C of 72b1610bcd54798702c9c5481e814b4c into c3cb81ec529341b2bb17682f4482e3bc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:40,414 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:40,414 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/C, priority=12, startTime=1732159480217; duration=0sec 2024-11-21T03:24:40,414 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:40,414 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:C 2024-11-21T03:24:40,417 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a492a031c7624bc28dbf198f05f527ed_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a492a031c7624bc28dbf198f05f527ed_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:40,418 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/3fd2d49249bd4b6c9fc74dc83ad64e67, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:40,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/3fd2d49249bd4b6c9fc74dc83ad64e67 is 175, key is test_row_0/A:col10/1732159479744/Put/seqid=0 2024-11-21T03:24:40,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741982_1158 (size=48389) 2024-11-21T03:24:40,430 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/3fd2d49249bd4b6c9fc74dc83ad64e67 2024-11-21T03:24:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T03:24:40,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/971f5d2caf38437ba38c524c778f4f23 is 50, key is test_row_0/B:col10/1732159479744/Put/seqid=0 2024-11-21T03:24:40,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159540459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159540459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159540459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159540460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741983_1159 (size=12151) 2024-11-21T03:24:40,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/971f5d2caf38437ba38c524c778f4f23 2024-11-21T03:24:40,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:40,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:40,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/70cd5278767b4c53bb16d21e22563d56 is 50, key is test_row_0/C:col10/1732159479744/Put/seqid=0 2024-11-21T03:24:40,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741984_1160 (size=12151) 2024-11-21T03:24:40,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/70cd5278767b4c53bb16d21e22563d56 2024-11-21T03:24:40,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/3fd2d49249bd4b6c9fc74dc83ad64e67 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3fd2d49249bd4b6c9fc74dc83ad64e67 2024-11-21T03:24:40,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3fd2d49249bd4b6c9fc74dc83ad64e67, entries=250, sequenceid=159, filesize=47.3 K 2024-11-21T03:24:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/971f5d2caf38437ba38c524c778f4f23 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/971f5d2caf38437ba38c524c778f4f23 2024-11-21T03:24:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/971f5d2caf38437ba38c524c778f4f23, entries=150, sequenceid=159, filesize=11.9 K 2024-11-21T03:24:40,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/70cd5278767b4c53bb16d21e22563d56 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/70cd5278767b4c53bb16d21e22563d56 2024-11-21T03:24:40,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/70cd5278767b4c53bb16d21e22563d56, entries=150, sequenceid=159, filesize=11.9 K 2024-11-21T03:24:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T03:24:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 72b1610bcd54798702c9c5481e814b4c in 336ms, sequenceid=159, compaction requested=false 2024-11-21T03:24:40,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:40,707 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:40,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:24:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c524cca77be74f7c9c3c46623df3ef2b_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159480342/Put/seqid=0 2024-11-21T03:24:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741985_1161 (size=12304) 2024-11-21T03:24:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:40,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:40,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:40,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159540918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159540926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159540928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159540941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159540941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:40,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:40,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:40,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:40,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T03:24:41,016 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T03:24:41,016 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T03:24:41,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159541048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159541049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159541057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159541057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:41,191 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c524cca77be74f7c9c3c46623df3ef2b_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c524cca77be74f7c9c3c46623df3ef2b_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:41,192 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/1ba5751855fc4791b4148bdd657878f5, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:41,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/1ba5751855fc4791b4148bdd657878f5 is 175, key is test_row_0/A:col10/1732159480342/Put/seqid=0 2024-11-21T03:24:41,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741986_1162 (size=31101) 2024-11-21T03:24:41,238 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/1ba5751855fc4791b4148bdd657878f5 2024-11-21T03:24:41,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/fc496ecc59f64864a2a8bac936810d89 is 50, key is test_row_0/B:col10/1732159480342/Put/seqid=0 2024-11-21T03:24:41,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159541269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159541269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159541269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159541285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741987_1163 (size=9757) 2024-11-21T03:24:41,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/fc496ecc59f64864a2a8bac936810d89 2024-11-21T03:24:41,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/2e6bf7cafdfd4301893ce1c89e65c3e3 is 50, key is test_row_0/C:col10/1732159480342/Put/seqid=0 2024-11-21T03:24:41,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741988_1164 (size=9757) 2024-11-21T03:24:41,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/2e6bf7cafdfd4301893ce1c89e65c3e3 2024-11-21T03:24:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/1ba5751855fc4791b4148bdd657878f5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/1ba5751855fc4791b4148bdd657878f5 2024-11-21T03:24:41,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/1ba5751855fc4791b4148bdd657878f5, entries=150, sequenceid=173, filesize=30.4 K 2024-11-21T03:24:41,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/fc496ecc59f64864a2a8bac936810d89 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/fc496ecc59f64864a2a8bac936810d89 2024-11-21T03:24:41,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/fc496ecc59f64864a2a8bac936810d89, entries=100, sequenceid=173, filesize=9.5 K 2024-11-21T03:24:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/2e6bf7cafdfd4301893ce1c89e65c3e3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2e6bf7cafdfd4301893ce1c89e65c3e3 2024-11-21T03:24:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T03:24:41,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2e6bf7cafdfd4301893ce1c89e65c3e3, entries=100, sequenceid=173, filesize=9.5 K 2024-11-21T03:24:41,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 72b1610bcd54798702c9c5481e814b4c in 778ms, sequenceid=173, compaction requested=true 2024-11-21T03:24:41,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:41,485 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:41,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:41,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:41,487 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:41,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:41,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:41,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:41,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:41,489 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:41,489 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/A is initiating minor compaction (all files) 2024-11-21T03:24:41,490 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/A in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,490 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8777862263f44cb5acd30211cf5acd11, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3fd2d49249bd4b6c9fc74dc83ad64e67, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/1ba5751855fc4791b4148bdd657878f5] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=108.3 K 2024-11-21T03:24:41,490 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,490 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8777862263f44cb5acd30211cf5acd11, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3fd2d49249bd4b6c9fc74dc83ad64e67, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/1ba5751855fc4791b4148bdd657878f5] 2024-11-21T03:24:41,491 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:41,491 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/B is initiating minor compaction (all files) 2024-11-21T03:24:41,491 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/B in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,491 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b30d17dc243840d3947e14be38a7c104, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/971f5d2caf38437ba38c524c778f4f23, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/fc496ecc59f64864a2a8bac936810d89] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=33.5 K 2024-11-21T03:24:41,491 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8777862263f44cb5acd30211cf5acd11, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159479133 2024-11-21T03:24:41,499 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting b30d17dc243840d3947e14be38a7c104, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159479133 2024-11-21T03:24:41,499 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fd2d49249bd4b6c9fc74dc83ad64e67, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732159479711 2024-11-21T03:24:41,506 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 971f5d2caf38437ba38c524c778f4f23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732159479711 2024-11-21T03:24:41,506 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting fc496ecc59f64864a2a8bac936810d89, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732159480342 2024-11-21T03:24:41,508 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ba5751855fc4791b4148bdd657878f5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732159480340 2024-11-21T03:24:41,520 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:41,521 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/57f0a692d502433cb68d82dd5018632c is 50, key is test_row_0/B:col10/1732159480342/Put/seqid=0 2024-11-21T03:24:41,537 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:41,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741989_1165 (size=12527) 2024-11-21T03:24:41,565 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411219b27c43307ac455997e5fc4a2d27d823_72b1610bcd54798702c9c5481e814b4c store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:41,568 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411219b27c43307ac455997e5fc4a2d27d823_72b1610bcd54798702c9c5481e814b4c, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:41,568 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219b27c43307ac455997e5fc4a2d27d823_72b1610bcd54798702c9c5481e814b4c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:41,589 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:41,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:24:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:41,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741990_1166 (size=4469) 2024-11-21T03:24:41,609 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#A#compaction#139 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:41,609 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/b6e7a148b6ee4eb2b838a3b9550724e1 is 175, key is test_row_0/A:col10/1732159480342/Put/seqid=0 2024-11-21T03:24:41,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217992765103604d488b1f2170bfbceb12_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159480939/Put/seqid=0 2024-11-21T03:24:41,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159541616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159541620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159541625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159541625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741991_1167 (size=31588) 2024-11-21T03:24:41,703 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/b6e7a148b6ee4eb2b838a3b9550724e1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b6e7a148b6ee4eb2b838a3b9550724e1 2024-11-21T03:24:41,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741992_1168 (size=12304) 2024-11-21T03:24:41,718 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/A of 72b1610bcd54798702c9c5481e814b4c into b6e7a148b6ee4eb2b838a3b9550724e1(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:41,718 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:41,718 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/A, priority=13, startTime=1732159481485; duration=0sec 2024-11-21T03:24:41,718 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:41,718 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:A 2024-11-21T03:24:41,718 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:41,720 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:41,720 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/C is initiating minor compaction (all files) 2024-11-21T03:24:41,720 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/C in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,721 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c3cb81ec529341b2bb17682f4482e3bc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/70cd5278767b4c53bb16d21e22563d56, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2e6bf7cafdfd4301893ce1c89e65c3e3] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=33.5 K 2024-11-21T03:24:41,722 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3cb81ec529341b2bb17682f4482e3bc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159479133 2024-11-21T03:24:41,722 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70cd5278767b4c53bb16d21e22563d56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732159479711 2024-11-21T03:24:41,722 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e6bf7cafdfd4301893ce1c89e65c3e3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732159480342 2024-11-21T03:24:41,738 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:41,739 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/ff848620d04540c0ace93d235951dc8a is 50, key is test_row_0/C:col10/1732159480342/Put/seqid=0 2024-11-21T03:24:41,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159541738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159541738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:41,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159541748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741993_1169 (size=12527) 2024-11-21T03:24:41,794 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/ff848620d04540c0ace93d235951dc8a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ff848620d04540c0ace93d235951dc8a 2024-11-21T03:24:41,804 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/C of 72b1610bcd54798702c9c5481e814b4c into ff848620d04540c0ace93d235951dc8a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:41,804 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:41,804 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/C, priority=13, startTime=1732159481487; duration=0sec 2024-11-21T03:24:41,804 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:41,805 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:C 2024-11-21T03:24:41,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:41,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:41,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159541948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159541948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159541948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,950 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in 72b1610bcd54798702c9c5481e814b4c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159541957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:41,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:41,973 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/57f0a692d502433cb68d82dd5018632c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/57f0a692d502433cb68d82dd5018632c 2024-11-21T03:24:41,980 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/B of 72b1610bcd54798702c9c5481e814b4c into 57f0a692d502433cb68d82dd5018632c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:41,980 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:41,980 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/B, priority=13, startTime=1732159481487; duration=0sec 2024-11-21T03:24:41,981 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:41,981 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:B 2024-11-21T03:24:42,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:42,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:42,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217992765103604d488b1f2170bfbceb12_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217992765103604d488b1f2170bfbceb12_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:42,142 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/dc4e454d7d4a43cd8bc4a99dd8741d8e, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:42,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/dc4e454d7d4a43cd8bc4a99dd8741d8e is 175, key is test_row_0/A:col10/1732159480939/Put/seqid=0 2024-11-21T03:24:42,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159542138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:42,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741994_1170 (size=31105) 2024-11-21T03:24:42,215 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/dc4e454d7d4a43cd8bc4a99dd8741d8e 2024-11-21T03:24:42,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:42,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:42,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/06e7c76402b4463ebc7230f740d836c3 is 50, key is test_row_0/B:col10/1732159480939/Put/seqid=0 2024-11-21T03:24:42,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741995_1171 (size=12151) 2024-11-21T03:24:42,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159542257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:42,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159542257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:42,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159542267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:42,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:42,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T03:24:42,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:42,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:42,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/06e7c76402b4463ebc7230f740d836c3 2024-11-21T03:24:42,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/2c4e5d0dc9e847dfafbc0839af03fb93 is 50, key is test_row_0/C:col10/1732159480939/Put/seqid=0 2024-11-21T03:24:42,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741996_1172 (size=12151) 2024-11-21T03:24:42,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/2c4e5d0dc9e847dfafbc0839af03fb93 2024-11-21T03:24:42,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:42,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/dc4e454d7d4a43cd8bc4a99dd8741d8e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/dc4e454d7d4a43cd8bc4a99dd8741d8e 2024-11-21T03:24:42,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:42,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/dc4e454d7d4a43cd8bc4a99dd8741d8e, entries=150, sequenceid=199, filesize=30.4 K 2024-11-21T03:24:42,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/06e7c76402b4463ebc7230f740d836c3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/06e7c76402b4463ebc7230f740d836c3 2024-11-21T03:24:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/06e7c76402b4463ebc7230f740d836c3, entries=150, sequenceid=199, filesize=11.9 K 2024-11-21T03:24:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/2c4e5d0dc9e847dfafbc0839af03fb93 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2c4e5d0dc9e847dfafbc0839af03fb93 2024-11-21T03:24:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2c4e5d0dc9e847dfafbc0839af03fb93, entries=150, sequenceid=199, filesize=11.9 K 2024-11-21T03:24:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 72b1610bcd54798702c9c5481e814b4c in 1131ms, sequenceid=199, compaction requested=false 2024-11-21T03:24:42,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:42,774 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:42,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:24:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121aa62648b4b86461c9ef4271348489b9b_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159482770/Put/seqid=0 2024-11-21T03:24:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741997_1173 (size=14794) 2024-11-21T03:24:42,837 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:42,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,846 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121aa62648b4b86461c9ef4271348489b9b_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121aa62648b4b86461c9ef4271348489b9b_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:42,847 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/8205c587e8bc4ce4ad7d1b871da16e4b, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:42,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/8205c587e8bc4ce4ad7d1b871da16e4b is 175, key is test_row_0/A:col10/1732159482770/Put/seqid=0 2024-11-21T03:24:42,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159542851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:42,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159542853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:42,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159542854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:42,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741998_1174 (size=39749) 2024-11-21T03:24:42,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159542959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:42,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159542963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:42,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159542968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:42,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:42,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:42,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:42,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:43,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:43,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159543148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:43,153 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_0' in 72b1610bcd54798702c9c5481e814b4c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:43,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159543167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159543169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:43,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159543178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:43,271 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/8205c587e8bc4ce4ad7d1b871da16e4b 2024-11-21T03:24:43,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/4b20a7a3138b4ddcb4309d179f0c199b is 50, key is test_row_0/B:col10/1732159482770/Put/seqid=0 2024-11-21T03:24:43,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:43,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:43,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741999_1175 (size=12151) 2024-11-21T03:24:43,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:43,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:43,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159543478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:43,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159543488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:43,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159543488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:43,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:43,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:43,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/4b20a7a3138b4ddcb4309d179f0c199b 2024-11-21T03:24:43,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/d9b99b9e91954546a5e8e82c627e4940 is 50, key is test_row_0/C:col10/1732159482770/Put/seqid=0 2024-11-21T03:24:43,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742000_1176 (size=12151) 2024-11-21T03:24:43,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:43,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:43,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:43,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:43,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:43,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:43,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159543977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:43,979 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in 72b1610bcd54798702c9c5481e814b4c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:43,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159543988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:43,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159543998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159543998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:44,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:44,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:44,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:44,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:44,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:44,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/d9b99b9e91954546a5e8e82c627e4940 2024-11-21T03:24:44,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/8205c587e8bc4ce4ad7d1b871da16e4b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8205c587e8bc4ce4ad7d1b871da16e4b 2024-11-21T03:24:44,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8205c587e8bc4ce4ad7d1b871da16e4b, entries=200, sequenceid=214, filesize=38.8 K 2024-11-21T03:24:44,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/4b20a7a3138b4ddcb4309d179f0c199b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4b20a7a3138b4ddcb4309d179f0c199b 2024-11-21T03:24:44,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4b20a7a3138b4ddcb4309d179f0c199b, entries=150, sequenceid=214, filesize=11.9 K 2024-11-21T03:24:44,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/d9b99b9e91954546a5e8e82c627e4940 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/d9b99b9e91954546a5e8e82c627e4940 2024-11-21T03:24:44,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:44,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:44,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:44,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:44,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:44,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:44,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/d9b99b9e91954546a5e8e82c627e4940, entries=150, sequenceid=214, filesize=11.9 K 2024-11-21T03:24:44,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 72b1610bcd54798702c9c5481e814b4c in 1461ms, sequenceid=214, compaction requested=true 2024-11-21T03:24:44,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:44,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:44,235 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:44,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:44,237 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:44,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:44,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:44,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:44,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:44,252 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:44,252 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:44,252 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/B is initiating minor compaction (all files) 2024-11-21T03:24:44,252 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/A is initiating minor compaction (all files) 2024-11-21T03:24:44,252 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/B in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:44,253 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/57f0a692d502433cb68d82dd5018632c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/06e7c76402b4463ebc7230f740d836c3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4b20a7a3138b4ddcb4309d179f0c199b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.0 K 2024-11-21T03:24:44,253 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/A in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:44,253 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b6e7a148b6ee4eb2b838a3b9550724e1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/dc4e454d7d4a43cd8bc4a99dd8741d8e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8205c587e8bc4ce4ad7d1b871da16e4b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=100.0 K 2024-11-21T03:24:44,253 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:44,253 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b6e7a148b6ee4eb2b838a3b9550724e1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/dc4e454d7d4a43cd8bc4a99dd8741d8e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8205c587e8bc4ce4ad7d1b871da16e4b] 2024-11-21T03:24:44,253 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 57f0a692d502433cb68d82dd5018632c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732159480319 2024-11-21T03:24:44,254 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6e7a148b6ee4eb2b838a3b9550724e1, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732159480319 2024-11-21T03:24:44,254 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 06e7c76402b4463ebc7230f740d836c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732159480904 2024-11-21T03:24:44,254 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc4e454d7d4a43cd8bc4a99dd8741d8e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732159480904 2024-11-21T03:24:44,254 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b20a7a3138b4ddcb4309d179f0c199b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159481617 2024-11-21T03:24:44,256 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8205c587e8bc4ce4ad7d1b871da16e4b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159481617 2024-11-21T03:24:44,263 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:44,281 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112108416ff5fa974589ab12d82d71e47236_72b1610bcd54798702c9c5481e814b4c store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:44,283 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112108416ff5fa974589ab12d82d71e47236_72b1610bcd54798702c9c5481e814b4c, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:44,283 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112108416ff5fa974589ab12d82d71e47236_72b1610bcd54798702c9c5481e814b4c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:44,287 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:44,287 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/f2a92e5c0c19471e904574cd1fa5f08d is 50, key is test_row_0/B:col10/1732159482770/Put/seqid=0 2024-11-21T03:24:44,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742001_1177 (size=4469) 2024-11-21T03:24:44,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742002_1178 (size=12629) 2024-11-21T03:24:44,331 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/f2a92e5c0c19471e904574cd1fa5f08d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f2a92e5c0c19471e904574cd1fa5f08d 2024-11-21T03:24:44,351 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/B of 72b1610bcd54798702c9c5481e814b4c into f2a92e5c0c19471e904574cd1fa5f08d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:44,351 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:44,351 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/B, priority=13, startTime=1732159484237; duration=0sec 2024-11-21T03:24:44,351 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:44,351 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:B 2024-11-21T03:24:44,352 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:44,358 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:44,359 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/C is initiating minor compaction (all files) 2024-11-21T03:24:44,359 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/C in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:44,359 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ff848620d04540c0ace93d235951dc8a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2c4e5d0dc9e847dfafbc0839af03fb93, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/d9b99b9e91954546a5e8e82c627e4940] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.0 K 2024-11-21T03:24:44,361 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting ff848620d04540c0ace93d235951dc8a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732159480319 2024-11-21T03:24:44,361 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c4e5d0dc9e847dfafbc0839af03fb93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732159480904 2024-11-21T03:24:44,362 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d9b99b9e91954546a5e8e82c627e4940, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159481617 2024-11-21T03:24:44,381 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#C#compaction#149 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:44,382 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/1223f1fc67094112ac68ed6f5bdff3a0 is 50, key is test_row_0/C:col10/1732159482770/Put/seqid=0 2024-11-21T03:24:44,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T03:24:44,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:44,389 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:24:44,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742003_1179 (size=12629) 2024-11-21T03:24:44,427 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/1223f1fc67094112ac68ed6f5bdff3a0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1223f1fc67094112ac68ed6f5bdff3a0 2024-11-21T03:24:44,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112130cb465ceee54209b3a81017e110a350_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159482852/Put/seqid=0 2024-11-21T03:24:44,442 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/C of 72b1610bcd54798702c9c5481e814b4c into 1223f1fc67094112ac68ed6f5bdff3a0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:44,442 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:44,442 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/C, priority=13, startTime=1732159484237; duration=0sec 2024-11-21T03:24:44,442 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:44,442 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:C 2024-11-21T03:24:44,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742004_1180 (size=12304) 2024-11-21T03:24:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:44,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112130cb465ceee54209b3a81017e110a350_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112130cb465ceee54209b3a81017e110a350_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/45259a28c11147eebbae49807b943995, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:44,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/45259a28c11147eebbae49807b943995 is 175, key is test_row_0/A:col10/1732159482852/Put/seqid=0 2024-11-21T03:24:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T03:24:44,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742005_1181 (size=31105) 2024-11-21T03:24:44,547 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/45259a28c11147eebbae49807b943995 2024-11-21T03:24:44,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/4e36f1a99c2b400c84e103e266aa6bd4 is 50, key is test_row_0/B:col10/1732159482852/Put/seqid=0 2024-11-21T03:24:44,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742006_1182 (size=12151) 2024-11-21T03:24:44,715 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#A#compaction#147 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:44,716 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/3d403b78303146e783dabbb953d53bf0 is 175, key is test_row_0/A:col10/1732159482770/Put/seqid=0 2024-11-21T03:24:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742007_1183 (size=31583) 2024-11-21T03:24:44,884 INFO [master/b69e2a1ee061:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-21T03:24:44,884 INFO [master/b69e2a1ee061:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-21T03:24:44,992 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/4e36f1a99c2b400c84e103e266aa6bd4 2024-11-21T03:24:45,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:45,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/ededa13df98945e1904be3c7e3c76726 is 50, key is test_row_0/C:col10/1732159482852/Put/seqid=0 2024-11-21T03:24:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742008_1184 (size=12151) 2024-11-21T03:24:45,035 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/ededa13df98945e1904be3c7e3c76726 2024-11-21T03:24:45,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/45259a28c11147eebbae49807b943995 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/45259a28c11147eebbae49807b943995 2024-11-21T03:24:45,048 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/45259a28c11147eebbae49807b943995, entries=150, sequenceid=239, filesize=30.4 K 2024-11-21T03:24:45,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/4e36f1a99c2b400c84e103e266aa6bd4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4e36f1a99c2b400c84e103e266aa6bd4 2024-11-21T03:24:45,055 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4e36f1a99c2b400c84e103e266aa6bd4, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T03:24:45,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/ededa13df98945e1904be3c7e3c76726 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ededa13df98945e1904be3c7e3c76726 2024-11-21T03:24:45,060 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ededa13df98945e1904be3c7e3c76726, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T03:24:45,086 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=53.67 KB/54960 for 72b1610bcd54798702c9c5481e814b4c in 698ms, sequenceid=239, compaction requested=false 2024-11-21T03:24:45,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:45,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:45,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-21T03:24:45,087 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:45,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-21T03:24:45,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:24:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:45,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-21T03:24:45,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.7450 sec 2024-11-21T03:24:45,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 4.7550 sec 2024-11-21T03:24:45,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121abc9a9687a0643258c75bf1e98e9662e_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159485053/Put/seqid=0 2024-11-21T03:24:45,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742009_1185 (size=14794) 2024-11-21T03:24:45,120 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:45,134 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/3d403b78303146e783dabbb953d53bf0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3d403b78303146e783dabbb953d53bf0 2024-11-21T03:24:45,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121abc9a9687a0643258c75bf1e98e9662e_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121abc9a9687a0643258c75bf1e98e9662e_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:45,138 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/d42c6771b1fc40f7b91e8dea6e8241d3, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:45,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/d42c6771b1fc40f7b91e8dea6e8241d3 is 175, key is test_row_0/A:col10/1732159485053/Put/seqid=0 2024-11-21T03:24:45,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159545144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159545146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159545149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742010_1186 (size=39749) 2024-11-21T03:24:45,171 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/A of 72b1610bcd54798702c9c5481e814b4c into 3d403b78303146e783dabbb953d53bf0(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:45,171 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:45,171 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/A, priority=13, startTime=1732159484235; duration=0sec 2024-11-21T03:24:45,171 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:45,171 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:A 2024-11-21T03:24:45,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159545168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,174 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_0' in 72b1610bcd54798702c9c5481e814b4c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159545261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159545261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159545272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159545479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159545480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159545488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,569 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/d42c6771b1fc40f7b91e8dea6e8241d3 2024-11-21T03:24:45,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/0bdc8c5aa3ce436eb228c74b3408e470 is 50, key is test_row_0/B:col10/1732159485053/Put/seqid=0 2024-11-21T03:24:45,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742011_1187 (size=12151) 2024-11-21T03:24:45,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159545798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159545798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159545798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:45,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/0bdc8c5aa3ce436eb228c74b3408e470 2024-11-21T03:24:46,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/def2046d16ae4db9bd4d4c9ba9b4cfe4 is 50, key is test_row_0/C:col10/1732159485053/Put/seqid=0 2024-11-21T03:24:46,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742012_1188 (size=12151) 2024-11-21T03:24:46,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/def2046d16ae4db9bd4d4c9ba9b4cfe4 2024-11-21T03:24:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/d42c6771b1fc40f7b91e8dea6e8241d3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/d42c6771b1fc40f7b91e8dea6e8241d3 2024-11-21T03:24:46,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/d42c6771b1fc40f7b91e8dea6e8241d3, entries=200, sequenceid=254, filesize=38.8 K 2024-11-21T03:24:46,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/0bdc8c5aa3ce436eb228c74b3408e470 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0bdc8c5aa3ce436eb228c74b3408e470 2024-11-21T03:24:46,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0bdc8c5aa3ce436eb228c74b3408e470, entries=150, sequenceid=254, filesize=11.9 K 2024-11-21T03:24:46,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/def2046d16ae4db9bd4d4c9ba9b4cfe4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/def2046d16ae4db9bd4d4c9ba9b4cfe4 2024-11-21T03:24:46,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/def2046d16ae4db9bd4d4c9ba9b4cfe4, entries=150, sequenceid=254, filesize=11.9 K 2024-11-21T03:24:46,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 72b1610bcd54798702c9c5481e814b4c in 1008ms, sequenceid=254, compaction requested=true 2024-11-21T03:24:46,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:46,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:46,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:46,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:46,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:46,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:46,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:24:46,096 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:46,096 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:46,106 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:46,106 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/B is initiating minor compaction (all files) 2024-11-21T03:24:46,106 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/B in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:46,107 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f2a92e5c0c19471e904574cd1fa5f08d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4e36f1a99c2b400c84e103e266aa6bd4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0bdc8c5aa3ce436eb228c74b3408e470] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.1 K 2024-11-21T03:24:46,107 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:46,107 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/A is initiating minor compaction (all files) 2024-11-21T03:24:46,107 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/A in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:46,107 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3d403b78303146e783dabbb953d53bf0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/45259a28c11147eebbae49807b943995, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/d42c6771b1fc40f7b91e8dea6e8241d3] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=100.0 K 2024-11-21T03:24:46,107 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:46,107 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3d403b78303146e783dabbb953d53bf0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/45259a28c11147eebbae49807b943995, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/d42c6771b1fc40f7b91e8dea6e8241d3] 2024-11-21T03:24:46,108 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a92e5c0c19471e904574cd1fa5f08d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159481617 2024-11-21T03:24:46,108 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e36f1a99c2b400c84e103e266aa6bd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732159482817 2024-11-21T03:24:46,108 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d403b78303146e783dabbb953d53bf0, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159481617 2024-11-21T03:24:46,108 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bdc8c5aa3ce436eb228c74b3408e470, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159485012 2024-11-21T03:24:46,108 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45259a28c11147eebbae49807b943995, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732159482817 2024-11-21T03:24:46,109 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d42c6771b1fc40f7b91e8dea6e8241d3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159485012 2024-11-21T03:24:46,123 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:46,124 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:46,125 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/eee5ce661eac4222a9ecf8b998b3f6e1 is 50, key is test_row_0/B:col10/1732159485053/Put/seqid=0 2024-11-21T03:24:46,145 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121c1587880298948fd8a56bc8edde0ae4d_72b1610bcd54798702c9c5481e814b4c store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:46,147 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121c1587880298948fd8a56bc8edde0ae4d_72b1610bcd54798702c9c5481e814b4c, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:46,147 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c1587880298948fd8a56bc8edde0ae4d_72b1610bcd54798702c9c5481e814b4c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:46,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742013_1189 (size=12731) 2024-11-21T03:24:46,163 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/eee5ce661eac4222a9ecf8b998b3f6e1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/eee5ce661eac4222a9ecf8b998b3f6e1 2024-11-21T03:24:46,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742014_1190 (size=4469) 2024-11-21T03:24:46,171 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#A#compaction#157 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:46,172 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/a3905411e315476baa458deb4af1df0f is 175, key is test_row_0/A:col10/1732159485053/Put/seqid=0 2024-11-21T03:24:46,179 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/B of 72b1610bcd54798702c9c5481e814b4c into eee5ce661eac4222a9ecf8b998b3f6e1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:46,179 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:46,179 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/B, priority=13, startTime=1732159486096; duration=0sec 2024-11-21T03:24:46,180 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:46,180 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:B 2024-11-21T03:24:46,180 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:46,181 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:46,181 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/C is initiating minor compaction (all files) 2024-11-21T03:24:46,181 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/C in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:46,181 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1223f1fc67094112ac68ed6f5bdff3a0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ededa13df98945e1904be3c7e3c76726, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/def2046d16ae4db9bd4d4c9ba9b4cfe4] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.1 K 2024-11-21T03:24:46,182 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1223f1fc67094112ac68ed6f5bdff3a0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159481617 2024-11-21T03:24:46,182 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting ededa13df98945e1904be3c7e3c76726, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732159482817 2024-11-21T03:24:46,183 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting def2046d16ae4db9bd4d4c9ba9b4cfe4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159485012 2024-11-21T03:24:46,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742015_1191 (size=31685) 2024-11-21T03:24:46,228 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#C#compaction#158 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:46,228 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/86ba620c647c4629a62ed47ace5b26a5 is 50, key is test_row_0/C:col10/1732159485053/Put/seqid=0 2024-11-21T03:24:46,236 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/a3905411e315476baa458deb4af1df0f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/a3905411e315476baa458deb4af1df0f 2024-11-21T03:24:46,247 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/A of 72b1610bcd54798702c9c5481e814b4c into a3905411e315476baa458deb4af1df0f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:46,247 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:46,247 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/A, priority=13, startTime=1732159486096; duration=0sec 2024-11-21T03:24:46,248 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:46,248 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:A 2024-11-21T03:24:46,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742016_1192 (size=12731) 2024-11-21T03:24:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:46,310 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:46,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:24:46,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159546377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159546380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159546382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:46,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121aa1c36b263c24300b0834615389c0e34_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159485139/Put/seqid=0 2024-11-21T03:24:46,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742017_1193 (size=12454) 2024-11-21T03:24:46,441 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:46,449 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121aa1c36b263c24300b0834615389c0e34_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121aa1c36b263c24300b0834615389c0e34_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:46,451 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/40a6f02ab79c4eb2b45469840297a4a6, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:46,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/40a6f02ab79c4eb2b45469840297a4a6 is 175, key is test_row_0/A:col10/1732159485139/Put/seqid=0 2024-11-21T03:24:46,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742018_1194 (size=31255) 2024-11-21T03:24:46,475 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/40a6f02ab79c4eb2b45469840297a4a6 2024-11-21T03:24:46,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/37a3a585575b4157aae6db220a742246 is 50, key is test_row_0/B:col10/1732159485139/Put/seqid=0 2024-11-21T03:24:46,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742019_1195 (size=12301) 2024-11-21T03:24:46,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:46,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159546498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159546498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:46,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159546498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:46,681 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/86ba620c647c4629a62ed47ace5b26a5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86ba620c647c4629a62ed47ace5b26a5 2024-11-21T03:24:46,689 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/C of 72b1610bcd54798702c9c5481e814b4c into 86ba620c647c4629a62ed47ace5b26a5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:46,689 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:46,690 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/C, priority=13, startTime=1732159486096; duration=0sec 2024-11-21T03:24:46,690 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:46,690 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:C 2024-11-21T03:24:46,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159546709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:46,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159546710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:46,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159546710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:46,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/37a3a585575b4157aae6db220a742246 2024-11-21T03:24:46,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/8020ccd00211426c8f44d3b0c74349f3 is 50, key is test_row_0/C:col10/1732159485139/Put/seqid=0 2024-11-21T03:24:46,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742020_1196 (size=12301) 2024-11-21T03:24:46,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/8020ccd00211426c8f44d3b0c74349f3 2024-11-21T03:24:46,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/40a6f02ab79c4eb2b45469840297a4a6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/40a6f02ab79c4eb2b45469840297a4a6 2024-11-21T03:24:47,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159547018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/40a6f02ab79c4eb2b45469840297a4a6, entries=150, sequenceid=280, filesize=30.5 K 2024-11-21T03:24:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:47,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/37a3a585575b4157aae6db220a742246 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/37a3a585575b4157aae6db220a742246 2024-11-21T03:24:47,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159547028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:47,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159547033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:47,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/37a3a585575b4157aae6db220a742246, entries=150, sequenceid=280, filesize=12.0 K 2024-11-21T03:24:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:47,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/8020ccd00211426c8f44d3b0c74349f3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8020ccd00211426c8f44d3b0c74349f3 2024-11-21T03:24:47,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8020ccd00211426c8f44d3b0c74349f3, entries=150, sequenceid=280, filesize=12.0 K 2024-11-21T03:24:47,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 72b1610bcd54798702c9c5481e814b4c in 746ms, sequenceid=280, compaction requested=false 2024-11-21T03:24:47,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:47,564 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:47,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:24:47,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121709ff7178ec44b3d829aac3cdbab08b7_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159487545/Put/seqid=0 2024-11-21T03:24:47,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742021_1197 (size=12454) 2024-11-21T03:24:47,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159547671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159547674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159547678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:47,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159547788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159547805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:47,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159547805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159548007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159548018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:48,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159548018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39154 deadline: 1732159548018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:48,024 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in 72b1610bcd54798702c9c5481e814b4c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,072 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:48,086 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121709ff7178ec44b3d829aac3cdbab08b7_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121709ff7178ec44b3d829aac3cdbab08b7_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:48,087 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/4e13a5906e324ab79f1cf087c916f6de, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:48,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/4e13a5906e324ab79f1cf087c916f6de is 175, key is test_row_0/A:col10/1732159487545/Put/seqid=0 2024-11-21T03:24:48,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742022_1198 (size=31255) 2024-11-21T03:24:48,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159548319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159548331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159548331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T03:24:48,530 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:48,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-21T03:24:48,534 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T03:24:48,535 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:48,535 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/4e13a5906e324ab79f1cf087c916f6de 2024-11-21T03:24:48,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:48,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/b0382c68bc61495790b83454e7a35ec2 is 50, key is test_row_0/B:col10/1732159487545/Put/seqid=0 2024-11-21T03:24:48,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742023_1199 (size=12301) 2024-11-21T03:24:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T03:24:48,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/b0382c68bc61495790b83454e7a35ec2 2024-11-21T03:24:48,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/86bf08d70245443681829d5c5b4504cb is 50, key is test_row_0/C:col10/1732159487545/Put/seqid=0 2024-11-21T03:24:48,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:48,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742024_1200 (size=12301) 2024-11-21T03:24:48,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/86bf08d70245443681829d5c5b4504cb 2024-11-21T03:24:48,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/4e13a5906e324ab79f1cf087c916f6de as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4e13a5906e324ab79f1cf087c916f6de 2024-11-21T03:24:48,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4e13a5906e324ab79f1cf087c916f6de, entries=150, sequenceid=297, filesize=30.5 K 2024-11-21T03:24:48,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/b0382c68bc61495790b83454e7a35ec2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b0382c68bc61495790b83454e7a35ec2 2024-11-21T03:24:48,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b0382c68bc61495790b83454e7a35ec2, entries=150, sequenceid=297, filesize=12.0 K 2024-11-21T03:24:48,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/86bf08d70245443681829d5c5b4504cb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86bf08d70245443681829d5c5b4504cb 2024-11-21T03:24:48,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86bf08d70245443681829d5c5b4504cb, entries=150, sequenceid=297, filesize=12.0 K 2024-11-21T03:24:48,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 72b1610bcd54798702c9c5481e814b4c in 1246ms, sequenceid=297, compaction requested=true 2024-11-21T03:24:48,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:48,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:48,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:48,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:48,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:24:48,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:48,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:24:48,810 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:48,810 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:48,812 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:48,812 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:48,812 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/C is initiating minor compaction (all files) 2024-11-21T03:24:48,812 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/A is initiating minor compaction (all files) 2024-11-21T03:24:48,812 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/A in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:48,812 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/C in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:48,812 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86ba620c647c4629a62ed47ace5b26a5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8020ccd00211426c8f44d3b0c74349f3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86bf08d70245443681829d5c5b4504cb] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.5 K 2024-11-21T03:24:48,812 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/a3905411e315476baa458deb4af1df0f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/40a6f02ab79c4eb2b45469840297a4a6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4e13a5906e324ab79f1cf087c916f6de] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=92.0 K 2024-11-21T03:24:48,812 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:48,812 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/a3905411e315476baa458deb4af1df0f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/40a6f02ab79c4eb2b45469840297a4a6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4e13a5906e324ab79f1cf087c916f6de] 2024-11-21T03:24:48,813 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3905411e315476baa458deb4af1df0f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159485012 2024-11-21T03:24:48,813 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ba620c647c4629a62ed47ace5b26a5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159485012 2024-11-21T03:24:48,813 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8020ccd00211426c8f44d3b0c74349f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732159485139 2024-11-21T03:24:48,814 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 86bf08d70245443681829d5c5b4504cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732159487545 2024-11-21T03:24:48,814 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40a6f02ab79c4eb2b45469840297a4a6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732159485139 2024-11-21T03:24:48,815 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e13a5906e324ab79f1cf087c916f6de, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732159487545 2024-11-21T03:24:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:48,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:48,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T03:24:48,833 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:48,838 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#C#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:48,838 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/415b0b91b3b449838e2e2de346f3f193 is 50, key is test_row_0/C:col10/1732159487545/Put/seqid=0 2024-11-21T03:24:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T03:24:48,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:48,852 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121468627263101449c8258c6398fc74ed3_72b1610bcd54798702c9c5481e814b4c store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:48,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:48,855 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121468627263101449c8258c6398fc74ed3_72b1610bcd54798702c9c5481e814b4c, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:48,855 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121468627263101449c8258c6398fc74ed3_72b1610bcd54798702c9c5481e814b4c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:48,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112182f28fd4865749c1a69666569b09a51c_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159488830/Put/seqid=0 2024-11-21T03:24:48,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742025_1201 (size=12983) 2024-11-21T03:24:48,904 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/415b0b91b3b449838e2e2de346f3f193 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/415b0b91b3b449838e2e2de346f3f193 2024-11-21T03:24:48,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742026_1202 (size=4469) 2024-11-21T03:24:48,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742027_1203 (size=14994) 2024-11-21T03:24:48,924 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/C of 72b1610bcd54798702c9c5481e814b4c into 415b0b91b3b449838e2e2de346f3f193(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:48,924 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:48,924 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/C, priority=13, startTime=1732159488810; duration=0sec 2024-11-21T03:24:48,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159548913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,930 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:48,930 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:C 2024-11-21T03:24:48,930 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:48,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159548929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:48,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159548929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:48,942 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:48,942 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/B is initiating minor compaction (all files) 2024-11-21T03:24:48,942 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/B in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:48,942 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/eee5ce661eac4222a9ecf8b998b3f6e1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/37a3a585575b4157aae6db220a742246, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b0382c68bc61495790b83454e7a35ec2] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.5 K 2024-11-21T03:24:48,943 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting eee5ce661eac4222a9ecf8b998b3f6e1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732159485012 2024-11-21T03:24:48,943 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 37a3a585575b4157aae6db220a742246, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732159485139 2024-11-21T03:24:48,944 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting b0382c68bc61495790b83454e7a35ec2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732159487545 2024-11-21T03:24:48,957 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:48,958 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/10118331cfd840a7ae47df97bab1425f is 50, key is test_row_0/B:col10/1732159487545/Put/seqid=0 2024-11-21T03:24:48,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742028_1204 (size=12983) 2024-11-21T03:24:49,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159549037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159549048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159549048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T03:24:49,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:49,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:49,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39158 deadline: 1732159549208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,211 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_0' in 72b1610bcd54798702c9c5481e814b4c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:24:49,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159549247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159549257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159549258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,317 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#A#compaction#165 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:49,318 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/cf2e662223b64fea996f54d1a5fb5be6 is 175, key is test_row_0/A:col10/1732159487545/Put/seqid=0 2024-11-21T03:24:49,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:49,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:49,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742029_1205 (size=31937) 2024-11-21T03:24:49,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,331 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112182f28fd4865749c1a69666569b09a51c_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182f28fd4865749c1a69666569b09a51c_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:49,332 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/2c01330deda54e04a40a31363e3dd337, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:49,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/2c01330deda54e04a40a31363e3dd337 is 175, key is test_row_0/A:col10/1732159488830/Put/seqid=0 2024-11-21T03:24:49,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742030_1206 (size=39949) 2024-11-21T03:24:49,342 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/2c01330deda54e04a40a31363e3dd337 2024-11-21T03:24:49,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/0d2f31b4796b4be79c5058ba2d23be77 is 50, key is test_row_0/B:col10/1732159488830/Put/seqid=0 2024-11-21T03:24:49,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742031_1207 (size=12301) 2024-11-21T03:24:49,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/0d2f31b4796b4be79c5058ba2d23be77 2024-11-21T03:24:49,389 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/10118331cfd840a7ae47df97bab1425f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/10118331cfd840a7ae47df97bab1425f 2024-11-21T03:24:49,400 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/B of 72b1610bcd54798702c9c5481e814b4c into 10118331cfd840a7ae47df97bab1425f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:49,400 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:49,400 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/B, priority=13, startTime=1732159488810; duration=0sec 2024-11-21T03:24:49,401 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:49,401 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:B 2024-11-21T03:24:49,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/b8fcb0e9f822482b8115aebe60ffef29 is 50, key is test_row_0/C:col10/1732159488830/Put/seqid=0 2024-11-21T03:24:49,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742032_1208 (size=12301) 2024-11-21T03:24:49,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/b8fcb0e9f822482b8115aebe60ffef29 2024-11-21T03:24:49,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/2c01330deda54e04a40a31363e3dd337 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/2c01330deda54e04a40a31363e3dd337 2024-11-21T03:24:49,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/2c01330deda54e04a40a31363e3dd337, entries=200, sequenceid=318, filesize=39.0 K 2024-11-21T03:24:49,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/0d2f31b4796b4be79c5058ba2d23be77 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0d2f31b4796b4be79c5058ba2d23be77 2024-11-21T03:24:49,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0d2f31b4796b4be79c5058ba2d23be77, entries=150, sequenceid=318, filesize=12.0 K 2024-11-21T03:24:49,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/b8fcb0e9f822482b8115aebe60ffef29 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/b8fcb0e9f822482b8115aebe60ffef29 2024-11-21T03:24:49,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:49,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:49,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/b8fcb0e9f822482b8115aebe60ffef29, entries=150, sequenceid=318, filesize=12.0 K 2024-11-21T03:24:49,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 72b1610bcd54798702c9c5481e814b4c in 668ms, sequenceid=318, compaction requested=false 2024-11-21T03:24:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:49,562 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:49,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T03:24:49,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121442448a61c2648ecb65137bb5dd311cd_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159489561/Put/seqid=0 2024-11-21T03:24:49,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742033_1209 (size=14994) 2024-11-21T03:24:49,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159549620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159549621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159549627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T03:24:49,729 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/cf2e662223b64fea996f54d1a5fb5be6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/cf2e662223b64fea996f54d1a5fb5be6 2024-11-21T03:24:49,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159549727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,736 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/A of 72b1610bcd54798702c9c5481e814b4c into cf2e662223b64fea996f54d1a5fb5be6(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:49,736 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:49,736 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/A, priority=13, startTime=1732159488810; duration=0sec 2024-11-21T03:24:49,736 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:49,736 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:A 2024-11-21T03:24:49,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159549738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159549739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:49,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:49,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159549940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159549948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159549948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:49,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:49,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:49,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:49,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,020 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:50,026 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121442448a61c2648ecb65137bb5dd311cd_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121442448a61c2648ecb65137bb5dd311cd_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:50,027 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/e04648f1a3a64aca89306391832b86ad, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:50,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/e04648f1a3a64aca89306391832b86ad is 175, key is test_row_0/A:col10/1732159489561/Put/seqid=0 2024-11-21T03:24:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742034_1210 (size=39949) 2024-11-21T03:24:50,074 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/e04648f1a3a64aca89306391832b86ad 2024-11-21T03:24:50,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/25d8a64fc0024b7781b0c8a7130c581e is 50, key is test_row_0/B:col10/1732159489561/Put/seqid=0 2024-11-21T03:24:50,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742035_1211 (size=12301) 2024-11-21T03:24:50,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159550248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:50,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159550258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:50,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159550259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:50,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:50,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:50,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/25d8a64fc0024b7781b0c8a7130c581e 2024-11-21T03:24:50,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/75fe0367a0e64fba87872b9807cd0dd1 is 50, key is test_row_0/C:col10/1732159489561/Put/seqid=0 2024-11-21T03:24:50,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742036_1212 (size=12301) 2024-11-21T03:24:50,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:50,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:50,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T03:24:50,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159550758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:50,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159550767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:50,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159550778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:50,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:50,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:50,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:50,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:50,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/75fe0367a0e64fba87872b9807cd0dd1 2024-11-21T03:24:50,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/e04648f1a3a64aca89306391832b86ad as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/e04648f1a3a64aca89306391832b86ad 2024-11-21T03:24:50,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/e04648f1a3a64aca89306391832b86ad, entries=200, sequenceid=337, filesize=39.0 K 2024-11-21T03:24:50,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/25d8a64fc0024b7781b0c8a7130c581e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/25d8a64fc0024b7781b0c8a7130c581e 2024-11-21T03:24:51,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/25d8a64fc0024b7781b0c8a7130c581e, entries=150, sequenceid=337, filesize=12.0 K 2024-11-21T03:24:51,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/75fe0367a0e64fba87872b9807cd0dd1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/75fe0367a0e64fba87872b9807cd0dd1 2024-11-21T03:24:51,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/75fe0367a0e64fba87872b9807cd0dd1, entries=150, sequenceid=337, filesize=12.0 K 2024-11-21T03:24:51,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 72b1610bcd54798702c9c5481e814b4c in 1462ms, sequenceid=337, compaction requested=true 2024-11-21T03:24:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:51,024 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:51,025 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:51,025 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/A is initiating minor compaction (all files) 2024-11-21T03:24:51,025 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/A in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:51,026 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/cf2e662223b64fea996f54d1a5fb5be6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/2c01330deda54e04a40a31363e3dd337, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/e04648f1a3a64aca89306391832b86ad] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=109.2 K 2024-11-21T03:24:51,026 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:51,026 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/cf2e662223b64fea996f54d1a5fb5be6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/2c01330deda54e04a40a31363e3dd337, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/e04648f1a3a64aca89306391832b86ad] 2024-11-21T03:24:51,026 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf2e662223b64fea996f54d1a5fb5be6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732159487545 2024-11-21T03:24:51,027 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c01330deda54e04a40a31363e3dd337, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732159487648 2024-11-21T03:24:51,027 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e04648f1a3a64aca89306391832b86ad, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732159488864 2024-11-21T03:24:51,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:51,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:51,027 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:51,028 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:51,028 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/B is initiating minor compaction (all files) 2024-11-21T03:24:51,029 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/B in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:51,029 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/10118331cfd840a7ae47df97bab1425f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0d2f31b4796b4be79c5058ba2d23be77, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/25d8a64fc0024b7781b0c8a7130c581e] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.7 K 2024-11-21T03:24:51,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:51,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:51,029 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 10118331cfd840a7ae47df97bab1425f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732159487545 2024-11-21T03:24:51,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:51,030 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2f31b4796b4be79c5058ba2d23be77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732159487672 2024-11-21T03:24:51,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:51,031 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d8a64fc0024b7781b0c8a7130c581e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732159488864 2024-11-21T03:24:51,049 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:51,067 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#B#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:51,067 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/8f1ec781b3fd43b1ab7f0a63c87bab4e is 50, key is test_row_0/B:col10/1732159489561/Put/seqid=0 2024-11-21T03:24:51,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T03:24:51,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:51,071 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T03:24:51,077 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121dfe385aaef7846d9a9122a85598ee983_72b1610bcd54798702c9c5481e814b4c store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:51,079 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121dfe385aaef7846d9a9122a85598ee983_72b1610bcd54798702c9c5481e814b4c, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:51,080 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dfe385aaef7846d9a9122a85598ee983_72b1610bcd54798702c9c5481e814b4c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:51,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742037_1213 (size=13085) 2024-11-21T03:24:51,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dc11afa9b7af450cb5f5aa9f2af4388f_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159489619/Put/seqid=0 2024-11-21T03:24:51,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742038_1214 (size=4469) 2024-11-21T03:24:51,184 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#A#compaction#174 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:51,196 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/c0dc17558cb7490885770025a3900824 is 175, key is test_row_0/A:col10/1732159489561/Put/seqid=0 2024-11-21T03:24:51,197 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/8f1ec781b3fd43b1ab7f0a63c87bab4e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/8f1ec781b3fd43b1ab7f0a63c87bab4e 2024-11-21T03:24:51,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742039_1215 (size=12454) 2024-11-21T03:24:51,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,215 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/B of 72b1610bcd54798702c9c5481e814b4c into 8f1ec781b3fd43b1ab7f0a63c87bab4e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:51,215 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:51,215 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/B, priority=13, startTime=1732159491027; duration=0sec 2024-11-21T03:24:51,215 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:51,215 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:B 2024-11-21T03:24:51,215 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:51,217 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dc11afa9b7af450cb5f5aa9f2af4388f_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dc11afa9b7af450cb5f5aa9f2af4388f_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:51,218 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:51,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/71233646bd0c435eb62938f2b1e1ffe0, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:51,218 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/C is initiating minor compaction (all files) 2024-11-21T03:24:51,218 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/C in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:51,218 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/415b0b91b3b449838e2e2de346f3f193, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/b8fcb0e9f822482b8115aebe60ffef29, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/75fe0367a0e64fba87872b9807cd0dd1] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.7 K 2024-11-21T03:24:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/71233646bd0c435eb62938f2b1e1ffe0 is 175, key is test_row_0/A:col10/1732159489619/Put/seqid=0 2024-11-21T03:24:51,222 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 415b0b91b3b449838e2e2de346f3f193, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732159487545 2024-11-21T03:24:51,223 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting b8fcb0e9f822482b8115aebe60ffef29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732159487672 2024-11-21T03:24:51,225 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 75fe0367a0e64fba87872b9807cd0dd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732159488864 2024-11-21T03:24:51,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742040_1216 (size=32039) 2024-11-21T03:24:51,234 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:51,235 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/1ecf2251958d4040bbd971302dd4e979 is 50, key is test_row_0/C:col10/1732159489561/Put/seqid=0 2024-11-21T03:24:51,250 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/c0dc17558cb7490885770025a3900824 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/c0dc17558cb7490885770025a3900824 2024-11-21T03:24:51,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742041_1217 (size=31255) 2024-11-21T03:24:51,260 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/A of 72b1610bcd54798702c9c5481e814b4c into c0dc17558cb7490885770025a3900824(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:51,260 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:51,260 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/A, priority=13, startTime=1732159491024; duration=0sec 2024-11-21T03:24:51,262 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:51,262 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:A 2024-11-21T03:24:51,264 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/71233646bd0c435eb62938f2b1e1ffe0 2024-11-21T03:24:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/77a7a5b95a3446d491febac3844d7358 is 50, key is test_row_0/B:col10/1732159489619/Put/seqid=0 2024-11-21T03:24:51,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742042_1218 (size=13085) 2024-11-21T03:24:51,308 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/1ecf2251958d4040bbd971302dd4e979 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1ecf2251958d4040bbd971302dd4e979 2024-11-21T03:24:51,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742043_1219 (size=12301) 2024-11-21T03:24:51,317 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/77a7a5b95a3446d491febac3844d7358 2024-11-21T03:24:51,329 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/C of 72b1610bcd54798702c9c5481e814b4c into 1ecf2251958d4040bbd971302dd4e979(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:51,330 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:51,330 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/C, priority=13, startTime=1732159491029; duration=0sec 2024-11-21T03:24:51,330 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:51,330 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:C 2024-11-21T03:24:51,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/7371bdee7493414dbcb3a0e93e6067d8 is 50, key is test_row_0/C:col10/1732159489619/Put/seqid=0 2024-11-21T03:24:51,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742044_1220 (size=12301) 2024-11-21T03:24:51,404 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/7371bdee7493414dbcb3a0e93e6067d8 2024-11-21T03:24:51,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/71233646bd0c435eb62938f2b1e1ffe0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/71233646bd0c435eb62938f2b1e1ffe0 2024-11-21T03:24:51,457 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/71233646bd0c435eb62938f2b1e1ffe0, entries=150, sequenceid=358, filesize=30.5 K 2024-11-21T03:24:51,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/77a7a5b95a3446d491febac3844d7358 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/77a7a5b95a3446d491febac3844d7358 2024-11-21T03:24:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,468 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/77a7a5b95a3446d491febac3844d7358, entries=150, sequenceid=358, filesize=12.0 K 2024-11-21T03:24:51,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/7371bdee7493414dbcb3a0e93e6067d8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/7371bdee7493414dbcb3a0e93e6067d8 2024-11-21T03:24:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,485 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/7371bdee7493414dbcb3a0e93e6067d8, entries=150, sequenceid=358, filesize=12.0 K 2024-11-21T03:24:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,493 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 72b1610bcd54798702c9c5481e814b4c in 421ms, sequenceid=358, compaction requested=false 2024-11-21T03:24:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-21T03:24:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-21T03:24:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-21T03:24:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9600 sec 2024-11-21T03:24:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.9680 sec 2024-11-21T03:24:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,875 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:51,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:24:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c544639a97814e5199bf08a144e0ce5c_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159491871/Put/seqid=0 2024-11-21T03:24:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742045_1221 (size=25158) 2024-11-21T03:24:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159551969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:51,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159551970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159551976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:51,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:51,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159552087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:52,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:52,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159552088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:52,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159552108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159552308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:52,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159552308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:52,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159552322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:52,376 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:52,412 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c544639a97814e5199bf08a144e0ce5c_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c544639a97814e5199bf08a144e0ce5c_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:52,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/4f50dece9bb5494396c557ff49483187, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:52,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/4f50dece9bb5494396c557ff49483187 is 175, key is test_row_0/A:col10/1732159491871/Put/seqid=0 2024-11-21T03:24:52,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742046_1222 (size=74795) 2024-11-21T03:24:52,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/4f50dece9bb5494396c557ff49483187 2024-11-21T03:24:52,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/e73fe7c9bf974c98b0560a34d423e0df is 50, key is test_row_0/B:col10/1732159491871/Put/seqid=0 2024-11-21T03:24:52,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742047_1223 (size=12301) 2024-11-21T03:24:52,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/e73fe7c9bf974c98b0560a34d423e0df 2024-11-21T03:24:52,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/1f1fb9127c50441184eb07606cc53e7b is 50, key is test_row_0/C:col10/1732159491871/Put/seqid=0 2024-11-21T03:24:52,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742048_1224 (size=12301) 2024-11-21T03:24:52,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159552617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:52,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159552630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:52,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159552640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T03:24:52,710 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:52,711 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:24:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-21T03:24:52,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:24:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:24:52,721 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:24:52,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:24:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:24:52,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T03:24:52,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:52,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:52,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:52,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:52,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:24:52,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/1f1fb9127c50441184eb07606cc53e7b 2024-11-21T03:24:52,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/4f50dece9bb5494396c557ff49483187 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4f50dece9bb5494396c557ff49483187 2024-11-21T03:24:52,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4f50dece9bb5494396c557ff49483187, entries=400, sequenceid=372, filesize=73.0 K 2024-11-21T03:24:52,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/e73fe7c9bf974c98b0560a34d423e0df as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/e73fe7c9bf974c98b0560a34d423e0df 2024-11-21T03:24:52,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/e73fe7c9bf974c98b0560a34d423e0df, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T03:24:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/1f1fb9127c50441184eb07606cc53e7b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1f1fb9127c50441184eb07606cc53e7b 2024-11-21T03:24:52,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1f1fb9127c50441184eb07606cc53e7b, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T03:24:52,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 72b1610bcd54798702c9c5481e814b4c in 1087ms, sequenceid=372, compaction requested=true 2024-11-21T03:24:52,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:52,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:52,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:52,963 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:52,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:52,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:52,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:52,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:24:52,964 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:52,970 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 138089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:52,970 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/A is initiating minor compaction (all files) 2024-11-21T03:24:52,970 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/A in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:52,970 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/c0dc17558cb7490885770025a3900824, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/71233646bd0c435eb62938f2b1e1ffe0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4f50dece9bb5494396c557ff49483187] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=134.9 K 2024-11-21T03:24:52,970 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:52,970 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/c0dc17558cb7490885770025a3900824, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/71233646bd0c435eb62938f2b1e1ffe0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4f50dece9bb5494396c557ff49483187] 2024-11-21T03:24:52,976 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:52,976 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/B is initiating minor compaction (all files) 2024-11-21T03:24:52,976 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/B in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:52,977 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/8f1ec781b3fd43b1ab7f0a63c87bab4e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/77a7a5b95a3446d491febac3844d7358, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/e73fe7c9bf974c98b0560a34d423e0df] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.8 K 2024-11-21T03:24:52,977 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting c0dc17558cb7490885770025a3900824, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732159488864 2024-11-21T03:24:52,979 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 71233646bd0c435eb62938f2b1e1ffe0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732159489616 2024-11-21T03:24:52,980 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f1ec781b3fd43b1ab7f0a63c87bab4e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732159488864 2024-11-21T03:24:52,981 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f50dece9bb5494396c557ff49483187, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732159491816 2024-11-21T03:24:52,984 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a7a5b95a3446d491febac3844d7358, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732159489616 2024-11-21T03:24:52,985 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e73fe7c9bf974c98b0560a34d423e0df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732159491852 2024-11-21T03:24:53,029 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:53,035 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#B#compaction#184 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:53,036 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/7d72474293af485fbc8f8733d3aee04f is 50, key is test_row_0/B:col10/1732159491871/Put/seqid=0 2024-11-21T03:24:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:24:53,043 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121c8de9b39d13f4538b1b4a50fcafed386_72b1610bcd54798702c9c5481e814b4c store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:53,047 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121c8de9b39d13f4538b1b4a50fcafed386_72b1610bcd54798702c9c5481e814b4c, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:53,047 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c8de9b39d13f4538b1b4a50fcafed386_72b1610bcd54798702c9c5481e814b4c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:53,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T03:24:53,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:53,052 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:24:53,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d8838a87de0e4be8b3bf81fd1fb6a8db_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159491963/Put/seqid=0 2024-11-21T03:24:53,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742049_1225 (size=13187) 2024-11-21T03:24:53,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. as already flushing 2024-11-21T03:24:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:53,145 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/7d72474293af485fbc8f8733d3aee04f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d72474293af485fbc8f8733d3aee04f 2024-11-21T03:24:53,153 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/B of 72b1610bcd54798702c9c5481e814b4c into 7d72474293af485fbc8f8733d3aee04f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:53,153 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:53,153 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/B, priority=13, startTime=1732159492963; duration=0sec 2024-11-21T03:24:53,153 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:53,153 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:B 2024-11-21T03:24:53,154 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:53,158 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:53,158 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/C is initiating minor compaction (all files) 2024-11-21T03:24:53,158 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/C in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:53,158 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1ecf2251958d4040bbd971302dd4e979, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/7371bdee7493414dbcb3a0e93e6067d8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1f1fb9127c50441184eb07606cc53e7b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.8 K 2024-11-21T03:24:53,159 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ecf2251958d4040bbd971302dd4e979, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732159488864 2024-11-21T03:24:53,159 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7371bdee7493414dbcb3a0e93e6067d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732159489616 2024-11-21T03:24:53,160 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f1fb9127c50441184eb07606cc53e7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732159491852 2024-11-21T03:24:53,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742050_1226 (size=4469) 2024-11-21T03:24:53,166 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#A#compaction#183 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:53,167 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/95e968244ef44abbb35d9ff1ea6ad8dc is 175, key is test_row_0/A:col10/1732159491871/Put/seqid=0 2024-11-21T03:24:53,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159553172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159553184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159553192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742051_1227 (size=12454) 2024-11-21T03:24:53,237 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:53,237 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/11cac57250f64730bfe9a3468bba56b5 is 50, key is test_row_0/C:col10/1732159491871/Put/seqid=0 2024-11-21T03:24:53,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742052_1228 (size=32141) 2024-11-21T03:24:53,265 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/95e968244ef44abbb35d9ff1ea6ad8dc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/95e968244ef44abbb35d9ff1ea6ad8dc 2024-11-21T03:24:53,272 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/A of 72b1610bcd54798702c9c5481e814b4c into 95e968244ef44abbb35d9ff1ea6ad8dc(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:53,272 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:53,272 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/A, priority=13, startTime=1732159492963; duration=0sec 2024-11-21T03:24:53,272 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:53,272 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:A 2024-11-21T03:24:53,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742053_1229 (size=13187) 2024-11-21T03:24:53,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159553298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159553298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159553308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:24:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159553508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159553508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159553520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:53,636 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d8838a87de0e4be8b3bf81fd1fb6a8db_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d8838a87de0e4be8b3bf81fd1fb6a8db_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:53,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/21d9f77f4e634da4a023e467bd0dfb81, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:53,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/21d9f77f4e634da4a023e467bd0dfb81 is 175, key is test_row_0/A:col10/1732159491963/Put/seqid=0 2024-11-21T03:24:53,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742054_1230 (size=31255) 2024-11-21T03:24:53,676 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/21d9f77f4e634da4a023e467bd0dfb81 2024-11-21T03:24:53,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/d3a0e5c3d5024f47a76e1bfde12e4185 is 50, key is test_row_0/B:col10/1732159491963/Put/seqid=0 2024-11-21T03:24:53,702 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/11cac57250f64730bfe9a3468bba56b5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/11cac57250f64730bfe9a3468bba56b5 2024-11-21T03:24:53,710 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/C of 72b1610bcd54798702c9c5481e814b4c into 11cac57250f64730bfe9a3468bba56b5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:53,710 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:53,710 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/C, priority=13, startTime=1732159492963; duration=0sec 2024-11-21T03:24:53,711 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:53,711 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:C 2024-11-21T03:24:53,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742055_1231 (size=12301) 2024-11-21T03:24:53,717 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/d3a0e5c3d5024f47a76e1bfde12e4185 2024-11-21T03:24:53,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/39f25d69345d420ebf060be4f32a97d4 is 50, key is test_row_0/C:col10/1732159491963/Put/seqid=0 2024-11-21T03:24:53,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742056_1232 (size=12301) 2024-11-21T03:24:53,785 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/39f25d69345d420ebf060be4f32a97d4 2024-11-21T03:24:53,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/21d9f77f4e634da4a023e467bd0dfb81 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/21d9f77f4e634da4a023e467bd0dfb81 2024-11-21T03:24:53,816 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/21d9f77f4e634da4a023e467bd0dfb81, entries=150, sequenceid=397, filesize=30.5 K 2024-11-21T03:24:53,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/d3a0e5c3d5024f47a76e1bfde12e4185 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d3a0e5c3d5024f47a76e1bfde12e4185 2024-11-21T03:24:53,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39180 deadline: 1732159553828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39178 deadline: 1732159553828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,833 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d3a0e5c3d5024f47a76e1bfde12e4185, entries=150, sequenceid=397, filesize=12.0 K 2024-11-21T03:24:53,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/39f25d69345d420ebf060be4f32a97d4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/39f25d69345d420ebf060be4f32a97d4 2024-11-21T03:24:53,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:24:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39184 deadline: 1732159553838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:24:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:24:53,848 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/39f25d69345d420ebf060be4f32a97d4, entries=150, sequenceid=397, filesize=12.0 K 2024-11-21T03:24:53,850 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 72b1610bcd54798702c9c5481e814b4c in 797ms, sequenceid=397, compaction requested=false 2024-11-21T03:24:53,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:53,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:53,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-21T03:24:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-21T03:24:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:24:53,862 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-21T03:24:53,862 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1380 sec 2024-11-21T03:24:53,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1520 sec 2024-11-21T03:24:54,059 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-21T03:24:54,059 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:54,059 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:54,060 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,061 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-21T03:24:54,061 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:54,061 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,061 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:54,063 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-21T03:24:54,063 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:54,063 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,063 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,063 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:54,068 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-21T03:24:54,068 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:54,068 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,068 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:54,339 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:24:54,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:24:54,339 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-21T03:24:54,339 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-11-21T03:24:54,339 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:54,340 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,340 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:54,340 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,340 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,340 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:54,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:54,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b258d9ea98d74e55ae71c00cae34112e_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159493179/Put/seqid=0 2024-11-21T03:24:54,368 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-21T03:24:54,368 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:54,368 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,368 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:54,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:54,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742057_1233 (size=12454) 2024-11-21T03:24:54,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:54,390 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b258d9ea98d74e55ae71c00cae34112e_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b258d9ea98d74e55ae71c00cae34112e_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:54,392 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/79d3ff7967da4a32931d0f8a1cf4e112, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:54,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/79d3ff7967da4a32931d0f8a1cf4e112 is 175, key is test_row_0/A:col10/1732159493179/Put/seqid=0 2024-11-21T03:24:54,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742058_1234 (size=31255) 2024-11-21T03:24:54,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/79d3ff7967da4a32931d0f8a1cf4e112 2024-11-21T03:24:54,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/7d5ea05bac3744e0a7d9973eba3bcaf3 is 50, key is test_row_0/B:col10/1732159493179/Put/seqid=0 2024-11-21T03:24:54,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742059_1235 (size=12301) 2024-11-21T03:24:54,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/7d5ea05bac3744e0a7d9973eba3bcaf3 2024-11-21T03:24:54,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/e3b4474ecc9d49e18f0bbf1ad1741136 is 50, key is test_row_0/C:col10/1732159493179/Put/seqid=0 2024-11-21T03:24:54,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742060_1236 (size=12301) 2024-11-21T03:24:54,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/e3b4474ecc9d49e18f0bbf1ad1741136 2024-11-21T03:24:54,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/79d3ff7967da4a32931d0f8a1cf4e112 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/79d3ff7967da4a32931d0f8a1cf4e112 2024-11-21T03:24:54,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/79d3ff7967da4a32931d0f8a1cf4e112, entries=150, sequenceid=413, filesize=30.5 K 2024-11-21T03:24:54,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/7d5ea05bac3744e0a7d9973eba3bcaf3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d5ea05bac3744e0a7d9973eba3bcaf3 2024-11-21T03:24:54,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d5ea05bac3744e0a7d9973eba3bcaf3, entries=150, sequenceid=413, filesize=12.0 K 2024-11-21T03:24:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/e3b4474ecc9d49e18f0bbf1ad1741136 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/e3b4474ecc9d49e18f0bbf1ad1741136 2024-11-21T03:24:54,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/e3b4474ecc9d49e18f0bbf1ad1741136, entries=150, sequenceid=413, filesize=12.0 K 2024-11-21T03:24:54,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=6.71 KB/6870 for 72b1610bcd54798702c9c5481e814b4c in 525ms, sequenceid=413, compaction requested=true 2024-11-21T03:24:54,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:54,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:24:54,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:54,864 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:54,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:24:54,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:54,864 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:54,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b1610bcd54798702c9c5481e814b4c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:24:54,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:54,865 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:54,865 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:54,865 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/A is initiating minor compaction (all files) 2024-11-21T03:24:54,865 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/B is initiating minor compaction (all files) 2024-11-21T03:24:54,865 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/A in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:54,865 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/B in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:54,865 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/95e968244ef44abbb35d9ff1ea6ad8dc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/21d9f77f4e634da4a023e467bd0dfb81, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/79d3ff7967da4a32931d0f8a1cf4e112] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=92.4 K 2024-11-21T03:24:54,865 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d72474293af485fbc8f8733d3aee04f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d3a0e5c3d5024f47a76e1bfde12e4185, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d5ea05bac3744e0a7d9973eba3bcaf3] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.9 K 2024-11-21T03:24:54,865 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:54,865 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/95e968244ef44abbb35d9ff1ea6ad8dc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/21d9f77f4e634da4a023e467bd0dfb81, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/79d3ff7967da4a32931d0f8a1cf4e112] 2024-11-21T03:24:54,866 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95e968244ef44abbb35d9ff1ea6ad8dc, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732159491852 2024-11-21T03:24:54,866 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d72474293af485fbc8f8733d3aee04f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732159491852 2024-11-21T03:24:54,866 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d3a0e5c3d5024f47a76e1bfde12e4185, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732159491963 2024-11-21T03:24:54,866 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d9f77f4e634da4a023e467bd0dfb81, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732159491963 2024-11-21T03:24:54,866 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d5ea05bac3744e0a7d9973eba3bcaf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732159493179 2024-11-21T03:24:54,866 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79d3ff7967da4a32931d0f8a1cf4e112, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732159493179 2024-11-21T03:24:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:24:54,867 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:24:54,872 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:54,873 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#B#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:54,874 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/67631d52f96f47d5bce915c5f725fb23 is 50, key is test_row_0/B:col10/1732159493179/Put/seqid=0 2024-11-21T03:24:54,876 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121727c65cdceaf4c9c9edfa3b5ec673a5d_72b1610bcd54798702c9c5481e814b4c store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:54,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742061_1237 (size=13289) 2024-11-21T03:24:54,894 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121727c65cdceaf4c9c9edfa3b5ec673a5d_72b1610bcd54798702c9c5481e814b4c, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:54,894 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121727c65cdceaf4c9c9edfa3b5ec673a5d_72b1610bcd54798702c9c5481e814b4c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:54,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742062_1238 (size=4469) 2024-11-21T03:24:54,904 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#A#compaction#193 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:54,904 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/23ca5e712ce34ee6b88a0d85f8e35c82 is 175, key is test_row_0/A:col10/1732159493179/Put/seqid=0 2024-11-21T03:24:54,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742063_1239 (size=32243) 2024-11-21T03:24:55,288 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/67631d52f96f47d5bce915c5f725fb23 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/67631d52f96f47d5bce915c5f725fb23 2024-11-21T03:24:55,295 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/B of 72b1610bcd54798702c9c5481e814b4c into 67631d52f96f47d5bce915c5f725fb23(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:55,295 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:55,295 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/B, priority=13, startTime=1732159494864; duration=0sec 2024-11-21T03:24:55,295 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:24:55,295 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:B 2024-11-21T03:24:55,295 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:24:55,299 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:24:55,299 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 72b1610bcd54798702c9c5481e814b4c/C is initiating minor compaction (all files) 2024-11-21T03:24:55,299 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b1610bcd54798702c9c5481e814b4c/C in TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:55,299 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/11cac57250f64730bfe9a3468bba56b5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/39f25d69345d420ebf060be4f32a97d4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/e3b4474ecc9d49e18f0bbf1ad1741136] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp, totalSize=36.9 K 2024-11-21T03:24:55,300 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 11cac57250f64730bfe9a3468bba56b5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732159491852 2024-11-21T03:24:55,300 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 39f25d69345d420ebf060be4f32a97d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732159491963 2024-11-21T03:24:55,301 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e3b4474ecc9d49e18f0bbf1ad1741136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732159493179 2024-11-21T03:24:55,318 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b1610bcd54798702c9c5481e814b4c#C#compaction#194 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:24:55,319 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/90f3a5c9945e4089a9acf4b6720c4cb4 is 50, key is test_row_0/C:col10/1732159493179/Put/seqid=0 2024-11-21T03:24:55,337 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/23ca5e712ce34ee6b88a0d85f8e35c82 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/23ca5e712ce34ee6b88a0d85f8e35c82 2024-11-21T03:24:55,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742064_1240 (size=13289) 2024-11-21T03:24:55,354 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/A of 72b1610bcd54798702c9c5481e814b4c into 23ca5e712ce34ee6b88a0d85f8e35c82(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:55,354 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:55,354 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/A, priority=13, startTime=1732159494864; duration=0sec 2024-11-21T03:24:55,355 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:55,355 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:A 2024-11-21T03:24:55,358 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/90f3a5c9945e4089a9acf4b6720c4cb4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/90f3a5c9945e4089a9acf4b6720c4cb4 2024-11-21T03:24:55,363 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b1610bcd54798702c9c5481e814b4c/C of 72b1610bcd54798702c9c5481e814b4c into 90f3a5c9945e4089a9acf4b6720c4cb4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:24:55,363 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b1610bcd54798702c9c5481e814b4c: 2024-11-21T03:24:55,363 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., storeName=72b1610bcd54798702c9c5481e814b4c/C, priority=13, startTime=1732159494864; duration=0sec 2024-11-21T03:24:55,363 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:24:55,364 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b1610bcd54798702c9c5481e814b4c:C 2024-11-21T03:24:58,119 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-11-21T03:24:58,119 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:58,119 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:58,120 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:58,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:59,298 INFO [Thread-619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-619. 2024-11-21T03:24:59,299 DEBUG [Thread-619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:24:59,299 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:59,299 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:59,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2394 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2320 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1097 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3291 rows 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1101 2024-11-21T03:24:59,299 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3301 rows 2024-11-21T03:24:59,299 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:24:59,299 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:24:59,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:59,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:24:59,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:24:59,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T03:24:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T03:24:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:24:59,303 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159499302"}]},"ts":"1732159499302"} 2024-11-21T03:24:59,304 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T03:24:59,304 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T03:24:59,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:24:59,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, UNASSIGN}] 2024-11-21T03:24:59,307 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, UNASSIGN 2024-11-21T03:24:59,307 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=72b1610bcd54798702c9c5481e814b4c, regionState=CLOSING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:24:59,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, UNASSIGN because future has completed 2024-11-21T03:24:59,309 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:24:59,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:24:59,317 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T03:24:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:24:59,462 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:59,462 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:24:59,462 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 72b1610bcd54798702c9c5481e814b4c, disabling compactions & flushes 2024-11-21T03:24:59,462 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:59,462 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:59,462 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. after waiting 0 ms 2024-11-21T03:24:59,462 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:24:59,462 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 72b1610bcd54798702c9c5481e814b4c 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-21T03:24:59,468 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ec2d5967b9bd45cfab338b26385ab549_72b1610bcd54798702c9c5481e814b4c is 50, key is test_row_0/A:col10/1732159499297/Put/seqid=0 2024-11-21T03:24:59,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742065_1241 (size=9914) 2024-11-21T03:24:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:24:59,874 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:24:59,882 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ec2d5967b9bd45cfab338b26385ab549_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ec2d5967b9bd45cfab338b26385ab549_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:24:59,884 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/caad575adacd48c19adce1238d0f64a0, store: [table=TestAcidGuarantees family=A region=72b1610bcd54798702c9c5481e814b4c] 2024-11-21T03:24:59,885 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/caad575adacd48c19adce1238d0f64a0 is 175, key is test_row_0/A:col10/1732159499297/Put/seqid=0 2024-11-21T03:24:59,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742066_1242 (size=22561) 2024-11-21T03:24:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:25:00,296 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/caad575adacd48c19adce1238d0f64a0 2024-11-21T03:25:00,308 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/6e82cea339894d5baceeab94ef78a028 is 50, key is test_row_0/B:col10/1732159499297/Put/seqid=0 2024-11-21T03:25:00,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742067_1243 (size=9857) 2024-11-21T03:25:00,315 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/6e82cea339894d5baceeab94ef78a028 2024-11-21T03:25:00,325 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/8f5a2e2e20714316b36ea9daffa638d4 is 50, key is test_row_0/C:col10/1732159499297/Put/seqid=0 2024-11-21T03:25:00,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742068_1244 (size=9857) 2024-11-21T03:25:00,336 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/8f5a2e2e20714316b36ea9daffa638d4 2024-11-21T03:25:00,342 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/A/caad575adacd48c19adce1238d0f64a0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/caad575adacd48c19adce1238d0f64a0 2024-11-21T03:25:00,348 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/caad575adacd48c19adce1238d0f64a0, entries=100, sequenceid=422, filesize=22.0 K 2024-11-21T03:25:00,351 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/B/6e82cea339894d5baceeab94ef78a028 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/6e82cea339894d5baceeab94ef78a028 2024-11-21T03:25:00,357 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/6e82cea339894d5baceeab94ef78a028, entries=100, sequenceid=422, filesize=9.6 K 2024-11-21T03:25:00,358 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/.tmp/C/8f5a2e2e20714316b36ea9daffa638d4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8f5a2e2e20714316b36ea9daffa638d4 2024-11-21T03:25:00,364 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8f5a2e2e20714316b36ea9daffa638d4, entries=100, sequenceid=422, filesize=9.6 K 2024-11-21T03:25:00,366 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 72b1610bcd54798702c9c5481e814b4c in 904ms, sequenceid=422, compaction requested=false 2024-11-21T03:25:00,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/5d7779b05dd5414d86bcb2a91f84f5ff, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8ac66fb50a874f9ea3f31c8be864223e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/aa2c73b3570840d394507c3488bb9eac, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/df3e6e29252c476987ed0bdbb238d473, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/649a98816e5d412c945072ea960fc695, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b54067b87d7d4a69ae4ed3e01f33fed2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/04615b0b1cad4b1487a497c84fb90ff0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/ba15576c3e1349e1b68fc5064dbff875, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8777862263f44cb5acd30211cf5acd11, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3fd2d49249bd4b6c9fc74dc83ad64e67, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b6e7a148b6ee4eb2b838a3b9550724e1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/1ba5751855fc4791b4148bdd657878f5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/dc4e454d7d4a43cd8bc4a99dd8741d8e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8205c587e8bc4ce4ad7d1b871da16e4b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3d403b78303146e783dabbb953d53bf0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/45259a28c11147eebbae49807b943995, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/d42c6771b1fc40f7b91e8dea6e8241d3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/a3905411e315476baa458deb4af1df0f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/40a6f02ab79c4eb2b45469840297a4a6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/cf2e662223b64fea996f54d1a5fb5be6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4e13a5906e324ab79f1cf087c916f6de, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/2c01330deda54e04a40a31363e3dd337, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/e04648f1a3a64aca89306391832b86ad, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/c0dc17558cb7490885770025a3900824, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/71233646bd0c435eb62938f2b1e1ffe0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4f50dece9bb5494396c557ff49483187, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/95e968244ef44abbb35d9ff1ea6ad8dc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/21d9f77f4e634da4a023e467bd0dfb81, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/79d3ff7967da4a32931d0f8a1cf4e112] to archive 2024-11-21T03:25:00,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:25:00,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/5d7779b05dd5414d86bcb2a91f84f5ff to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/5d7779b05dd5414d86bcb2a91f84f5ff 2024-11-21T03:25:00,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8ac66fb50a874f9ea3f31c8be864223e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8ac66fb50a874f9ea3f31c8be864223e 2024-11-21T03:25:00,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/aa2c73b3570840d394507c3488bb9eac to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/aa2c73b3570840d394507c3488bb9eac 2024-11-21T03:25:00,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/df3e6e29252c476987ed0bdbb238d473 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/df3e6e29252c476987ed0bdbb238d473 2024-11-21T03:25:00,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/649a98816e5d412c945072ea960fc695 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/649a98816e5d412c945072ea960fc695 2024-11-21T03:25:00,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b54067b87d7d4a69ae4ed3e01f33fed2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b54067b87d7d4a69ae4ed3e01f33fed2 2024-11-21T03:25:00,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/04615b0b1cad4b1487a497c84fb90ff0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/04615b0b1cad4b1487a497c84fb90ff0 2024-11-21T03:25:00,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/ba15576c3e1349e1b68fc5064dbff875 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/ba15576c3e1349e1b68fc5064dbff875 2024-11-21T03:25:00,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8777862263f44cb5acd30211cf5acd11 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8777862263f44cb5acd30211cf5acd11 2024-11-21T03:25:00,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3fd2d49249bd4b6c9fc74dc83ad64e67 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3fd2d49249bd4b6c9fc74dc83ad64e67 2024-11-21T03:25:00,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b6e7a148b6ee4eb2b838a3b9550724e1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/b6e7a148b6ee4eb2b838a3b9550724e1 2024-11-21T03:25:00,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/1ba5751855fc4791b4148bdd657878f5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/1ba5751855fc4791b4148bdd657878f5 2024-11-21T03:25:00,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/dc4e454d7d4a43cd8bc4a99dd8741d8e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/dc4e454d7d4a43cd8bc4a99dd8741d8e 2024-11-21T03:25:00,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8205c587e8bc4ce4ad7d1b871da16e4b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/8205c587e8bc4ce4ad7d1b871da16e4b 2024-11-21T03:25:00,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3d403b78303146e783dabbb953d53bf0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/3d403b78303146e783dabbb953d53bf0 2024-11-21T03:25:00,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/45259a28c11147eebbae49807b943995 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/45259a28c11147eebbae49807b943995 2024-11-21T03:25:00,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/d42c6771b1fc40f7b91e8dea6e8241d3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/d42c6771b1fc40f7b91e8dea6e8241d3 2024-11-21T03:25:00,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/a3905411e315476baa458deb4af1df0f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/a3905411e315476baa458deb4af1df0f 2024-11-21T03:25:00,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/40a6f02ab79c4eb2b45469840297a4a6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/40a6f02ab79c4eb2b45469840297a4a6 2024-11-21T03:25:00,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/cf2e662223b64fea996f54d1a5fb5be6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/cf2e662223b64fea996f54d1a5fb5be6 2024-11-21T03:25:00,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4e13a5906e324ab79f1cf087c916f6de to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4e13a5906e324ab79f1cf087c916f6de 2024-11-21T03:25:00,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/2c01330deda54e04a40a31363e3dd337 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/2c01330deda54e04a40a31363e3dd337 2024-11-21T03:25:00,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/e04648f1a3a64aca89306391832b86ad to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/e04648f1a3a64aca89306391832b86ad 2024-11-21T03:25:00,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/c0dc17558cb7490885770025a3900824 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/c0dc17558cb7490885770025a3900824 2024-11-21T03:25:00,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/71233646bd0c435eb62938f2b1e1ffe0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/71233646bd0c435eb62938f2b1e1ffe0 2024-11-21T03:25:00,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4f50dece9bb5494396c557ff49483187 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/4f50dece9bb5494396c557ff49483187 2024-11-21T03:25:00,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/95e968244ef44abbb35d9ff1ea6ad8dc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/95e968244ef44abbb35d9ff1ea6ad8dc 2024-11-21T03:25:00,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/21d9f77f4e634da4a023e467bd0dfb81 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/21d9f77f4e634da4a023e467bd0dfb81 2024-11-21T03:25:00,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/79d3ff7967da4a32931d0f8a1cf4e112 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/79d3ff7967da4a32931d0f8a1cf4e112 2024-11-21T03:25:00,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/6f47862db40344359592c23afee79ab9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d2d9f52c8a2f43b2bf2ee45ff8e0ef2b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7dc7a4cc19304aaaae08725c32aa2e18, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/5ed5f65a43e4475092f703108cdb897b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f023cd0d553645648da2d7e17530bffc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/940e69c397dc4c15ac58d68537a8a81d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4424cdf1ec2d41c794bccda89bb6adfa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b30d17dc243840d3947e14be38a7c104, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/899829f77bc546e093c6232ae05dd6fd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/971f5d2caf38437ba38c524c778f4f23, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/57f0a692d502433cb68d82dd5018632c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/fc496ecc59f64864a2a8bac936810d89, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/06e7c76402b4463ebc7230f740d836c3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f2a92e5c0c19471e904574cd1fa5f08d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4b20a7a3138b4ddcb4309d179f0c199b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4e36f1a99c2b400c84e103e266aa6bd4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/eee5ce661eac4222a9ecf8b998b3f6e1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0bdc8c5aa3ce436eb228c74b3408e470, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/37a3a585575b4157aae6db220a742246, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/10118331cfd840a7ae47df97bab1425f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b0382c68bc61495790b83454e7a35ec2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0d2f31b4796b4be79c5058ba2d23be77, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/8f1ec781b3fd43b1ab7f0a63c87bab4e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/25d8a64fc0024b7781b0c8a7130c581e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/77a7a5b95a3446d491febac3844d7358, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d72474293af485fbc8f8733d3aee04f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/e73fe7c9bf974c98b0560a34d423e0df, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d3a0e5c3d5024f47a76e1bfde12e4185, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d5ea05bac3744e0a7d9973eba3bcaf3] to archive 2024-11-21T03:25:00,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:25:00,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/6f47862db40344359592c23afee79ab9 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/6f47862db40344359592c23afee79ab9 2024-11-21T03:25:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:25:00,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d2d9f52c8a2f43b2bf2ee45ff8e0ef2b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d2d9f52c8a2f43b2bf2ee45ff8e0ef2b 2024-11-21T03:25:00,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7dc7a4cc19304aaaae08725c32aa2e18 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7dc7a4cc19304aaaae08725c32aa2e18 2024-11-21T03:25:00,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/5ed5f65a43e4475092f703108cdb897b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/5ed5f65a43e4475092f703108cdb897b 2024-11-21T03:25:00,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f023cd0d553645648da2d7e17530bffc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f023cd0d553645648da2d7e17530bffc 2024-11-21T03:25:00,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/940e69c397dc4c15ac58d68537a8a81d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/940e69c397dc4c15ac58d68537a8a81d 2024-11-21T03:25:00,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4424cdf1ec2d41c794bccda89bb6adfa to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4424cdf1ec2d41c794bccda89bb6adfa 2024-11-21T03:25:00,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b30d17dc243840d3947e14be38a7c104 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b30d17dc243840d3947e14be38a7c104 2024-11-21T03:25:00,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/899829f77bc546e093c6232ae05dd6fd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/899829f77bc546e093c6232ae05dd6fd 2024-11-21T03:25:00,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/971f5d2caf38437ba38c524c778f4f23 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/971f5d2caf38437ba38c524c778f4f23 2024-11-21T03:25:00,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/57f0a692d502433cb68d82dd5018632c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/57f0a692d502433cb68d82dd5018632c 2024-11-21T03:25:00,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/fc496ecc59f64864a2a8bac936810d89 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/fc496ecc59f64864a2a8bac936810d89 2024-11-21T03:25:00,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/06e7c76402b4463ebc7230f740d836c3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/06e7c76402b4463ebc7230f740d836c3 2024-11-21T03:25:00,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f2a92e5c0c19471e904574cd1fa5f08d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/f2a92e5c0c19471e904574cd1fa5f08d 2024-11-21T03:25:00,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4b20a7a3138b4ddcb4309d179f0c199b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4b20a7a3138b4ddcb4309d179f0c199b 2024-11-21T03:25:00,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4e36f1a99c2b400c84e103e266aa6bd4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/4e36f1a99c2b400c84e103e266aa6bd4 2024-11-21T03:25:00,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/eee5ce661eac4222a9ecf8b998b3f6e1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/eee5ce661eac4222a9ecf8b998b3f6e1 2024-11-21T03:25:00,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0bdc8c5aa3ce436eb228c74b3408e470 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0bdc8c5aa3ce436eb228c74b3408e470 2024-11-21T03:25:00,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/37a3a585575b4157aae6db220a742246 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/37a3a585575b4157aae6db220a742246 2024-11-21T03:25:00,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/10118331cfd840a7ae47df97bab1425f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/10118331cfd840a7ae47df97bab1425f 2024-11-21T03:25:00,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b0382c68bc61495790b83454e7a35ec2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/b0382c68bc61495790b83454e7a35ec2 2024-11-21T03:25:00,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0d2f31b4796b4be79c5058ba2d23be77 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/0d2f31b4796b4be79c5058ba2d23be77 2024-11-21T03:25:00,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/8f1ec781b3fd43b1ab7f0a63c87bab4e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/8f1ec781b3fd43b1ab7f0a63c87bab4e 2024-11-21T03:25:00,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/25d8a64fc0024b7781b0c8a7130c581e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/25d8a64fc0024b7781b0c8a7130c581e 2024-11-21T03:25:00,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/77a7a5b95a3446d491febac3844d7358 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/77a7a5b95a3446d491febac3844d7358 2024-11-21T03:25:00,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d72474293af485fbc8f8733d3aee04f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d72474293af485fbc8f8733d3aee04f 2024-11-21T03:25:00,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/e73fe7c9bf974c98b0560a34d423e0df to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/e73fe7c9bf974c98b0560a34d423e0df 2024-11-21T03:25:00,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d3a0e5c3d5024f47a76e1bfde12e4185 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/d3a0e5c3d5024f47a76e1bfde12e4185 2024-11-21T03:25:00,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d5ea05bac3744e0a7d9973eba3bcaf3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/7d5ea05bac3744e0a7d9973eba3bcaf3 2024-11-21T03:25:00,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/606f32f6ed2d4904ad8a987f0dc2201f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/4946f24ce0894c12aa5a7c82b7266574, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/f18b7573e3884533ad7fa735fd3dc2f0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8ef92fbf3f904e49a55bba85db7ee987, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/cabb5fe518474b5d9840019994d9d6c2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c9d32a2441994a81b12901d0b40c9c10, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/10309e96d80f4155ae3f9ce431753b03, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c3cb81ec529341b2bb17682f4482e3bc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/76dd3583748242f8974b94f560593bf6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/70cd5278767b4c53bb16d21e22563d56, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ff848620d04540c0ace93d235951dc8a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2e6bf7cafdfd4301893ce1c89e65c3e3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2c4e5d0dc9e847dfafbc0839af03fb93, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1223f1fc67094112ac68ed6f5bdff3a0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/d9b99b9e91954546a5e8e82c627e4940, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ededa13df98945e1904be3c7e3c76726, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86ba620c647c4629a62ed47ace5b26a5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/def2046d16ae4db9bd4d4c9ba9b4cfe4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8020ccd00211426c8f44d3b0c74349f3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/415b0b91b3b449838e2e2de346f3f193, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86bf08d70245443681829d5c5b4504cb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/b8fcb0e9f822482b8115aebe60ffef29, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1ecf2251958d4040bbd971302dd4e979, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/75fe0367a0e64fba87872b9807cd0dd1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/7371bdee7493414dbcb3a0e93e6067d8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/11cac57250f64730bfe9a3468bba56b5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1f1fb9127c50441184eb07606cc53e7b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/39f25d69345d420ebf060be4f32a97d4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/e3b4474ecc9d49e18f0bbf1ad1741136] to archive 2024-11-21T03:25:00,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:25:00,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/606f32f6ed2d4904ad8a987f0dc2201f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/606f32f6ed2d4904ad8a987f0dc2201f 2024-11-21T03:25:00,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/4946f24ce0894c12aa5a7c82b7266574 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/4946f24ce0894c12aa5a7c82b7266574 2024-11-21T03:25:00,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/f18b7573e3884533ad7fa735fd3dc2f0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/f18b7573e3884533ad7fa735fd3dc2f0 2024-11-21T03:25:00,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8ef92fbf3f904e49a55bba85db7ee987 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8ef92fbf3f904e49a55bba85db7ee987 2024-11-21T03:25:00,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/cabb5fe518474b5d9840019994d9d6c2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/cabb5fe518474b5d9840019994d9d6c2 2024-11-21T03:25:00,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c9d32a2441994a81b12901d0b40c9c10 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c9d32a2441994a81b12901d0b40c9c10 2024-11-21T03:25:00,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/10309e96d80f4155ae3f9ce431753b03 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/10309e96d80f4155ae3f9ce431753b03 2024-11-21T03:25:00,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c3cb81ec529341b2bb17682f4482e3bc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/c3cb81ec529341b2bb17682f4482e3bc 2024-11-21T03:25:00,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/76dd3583748242f8974b94f560593bf6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/76dd3583748242f8974b94f560593bf6 2024-11-21T03:25:00,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/70cd5278767b4c53bb16d21e22563d56 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/70cd5278767b4c53bb16d21e22563d56 2024-11-21T03:25:00,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ff848620d04540c0ace93d235951dc8a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ff848620d04540c0ace93d235951dc8a 2024-11-21T03:25:00,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2e6bf7cafdfd4301893ce1c89e65c3e3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2e6bf7cafdfd4301893ce1c89e65c3e3 2024-11-21T03:25:00,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2c4e5d0dc9e847dfafbc0839af03fb93 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/2c4e5d0dc9e847dfafbc0839af03fb93 2024-11-21T03:25:00,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1223f1fc67094112ac68ed6f5bdff3a0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1223f1fc67094112ac68ed6f5bdff3a0 2024-11-21T03:25:00,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/d9b99b9e91954546a5e8e82c627e4940 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/d9b99b9e91954546a5e8e82c627e4940 2024-11-21T03:25:00,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ededa13df98945e1904be3c7e3c76726 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/ededa13df98945e1904be3c7e3c76726 2024-11-21T03:25:00,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86ba620c647c4629a62ed47ace5b26a5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86ba620c647c4629a62ed47ace5b26a5 2024-11-21T03:25:00,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/def2046d16ae4db9bd4d4c9ba9b4cfe4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/def2046d16ae4db9bd4d4c9ba9b4cfe4 2024-11-21T03:25:00,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8020ccd00211426c8f44d3b0c74349f3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8020ccd00211426c8f44d3b0c74349f3 2024-11-21T03:25:00,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/415b0b91b3b449838e2e2de346f3f193 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/415b0b91b3b449838e2e2de346f3f193 2024-11-21T03:25:00,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86bf08d70245443681829d5c5b4504cb to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/86bf08d70245443681829d5c5b4504cb 2024-11-21T03:25:00,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/b8fcb0e9f822482b8115aebe60ffef29 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/b8fcb0e9f822482b8115aebe60ffef29 2024-11-21T03:25:00,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1ecf2251958d4040bbd971302dd4e979 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1ecf2251958d4040bbd971302dd4e979 2024-11-21T03:25:00,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/75fe0367a0e64fba87872b9807cd0dd1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/75fe0367a0e64fba87872b9807cd0dd1 2024-11-21T03:25:00,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/7371bdee7493414dbcb3a0e93e6067d8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/7371bdee7493414dbcb3a0e93e6067d8 2024-11-21T03:25:00,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/11cac57250f64730bfe9a3468bba56b5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/11cac57250f64730bfe9a3468bba56b5 2024-11-21T03:25:00,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1f1fb9127c50441184eb07606cc53e7b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/1f1fb9127c50441184eb07606cc53e7b 2024-11-21T03:25:00,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/39f25d69345d420ebf060be4f32a97d4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/39f25d69345d420ebf060be4f32a97d4 2024-11-21T03:25:00,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/e3b4474ecc9d49e18f0bbf1ad1741136 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/e3b4474ecc9d49e18f0bbf1ad1741136 2024-11-21T03:25:00,738 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/recovered.edits/425.seqid, newMaxSeqId=425, maxSeqId=4 2024-11-21T03:25:00,739 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. 2024-11-21T03:25:00,740 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 72b1610bcd54798702c9c5481e814b4c: Waiting for close lock at 1732159499462Running coprocessor pre-close hooks at 1732159499462Disabling compacts and flushes for region at 1732159499462Disabling writes for close at 1732159499462Obtaining lock to block concurrent updates at 1732159499462Preparing flush snapshotting stores in 72b1610bcd54798702c9c5481e814b4c at 1732159499462Finished memstore snapshotting TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732159499462Flushing stores of TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c. at 1732159499463 (+1 ms)Flushing 72b1610bcd54798702c9c5481e814b4c/A: creating writer at 1732159499463Flushing mob file 72b1610bcd54798702c9c5481e814b4c/A: appending metadata at 1732159499468 (+5 ms)Flushing mob file 72b1610bcd54798702c9c5481e814b4c/A: closing flushed file at 1732159499468Flushing 72b1610bcd54798702c9c5481e814b4c/A: appending metadata at 1732159499884 (+416 ms)Flushing 72b1610bcd54798702c9c5481e814b4c/A: closing flushed file at 1732159499884Flushing 72b1610bcd54798702c9c5481e814b4c/B: creating writer at 1732159500304 (+420 ms)Flushing 72b1610bcd54798702c9c5481e814b4c/B: appending metadata at 1732159500307 (+3 ms)Flushing 72b1610bcd54798702c9c5481e814b4c/B: closing flushed file at 1732159500308 (+1 ms)Flushing 72b1610bcd54798702c9c5481e814b4c/C: creating writer at 1732159500321 (+13 ms)Flushing 72b1610bcd54798702c9c5481e814b4c/C: appending metadata at 1732159500325 (+4 ms)Flushing 72b1610bcd54798702c9c5481e814b4c/C: closing flushed file at 1732159500325Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a4ef118: reopening flushed file at 1732159500341 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ce39f06: reopening flushed file at 1732159500348 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5501c08e: reopening flushed file at 1732159500357 (+9 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 72b1610bcd54798702c9c5481e814b4c in 904ms, sequenceid=422, compaction requested=false at 1732159500366 (+9 ms)Writing region close event to WAL at 1732159500732 (+366 ms)Running coprocessor post-close hooks at 1732159500739 (+7 ms)Closed at 1732159500739 2024-11-21T03:25:00,746 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:00,746 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=72b1610bcd54798702c9c5481e814b4c, regionState=CLOSED 2024-11-21T03:25:00,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:25:00,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-21T03:25:00,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 72b1610bcd54798702c9c5481e814b4c, server=b69e2a1ee061,45649,1732159441548 in 1.4440 sec 2024-11-21T03:25:00,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-21T03:25:00,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b1610bcd54798702c9c5481e814b4c, UNASSIGN in 1.4500 sec 2024-11-21T03:25:00,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-21T03:25:00,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4560 sec 2024-11-21T03:25:00,767 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159500766"}]},"ts":"1732159500766"} 2024-11-21T03:25:00,769 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T03:25:00,770 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T03:25:00,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4710 sec 2024-11-21T03:25:01,016 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T03:25:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:25:01,478 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:01,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T03:25:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:01,480 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T03:25:01,481 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:01,484 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,486 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/recovered.edits] 2024-11-21T03:25:01,488 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/23ca5e712ce34ee6b88a0d85f8e35c82 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/23ca5e712ce34ee6b88a0d85f8e35c82 2024-11-21T03:25:01,490 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/caad575adacd48c19adce1238d0f64a0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/A/caad575adacd48c19adce1238d0f64a0 2024-11-21T03:25:01,492 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/67631d52f96f47d5bce915c5f725fb23 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/67631d52f96f47d5bce915c5f725fb23 2024-11-21T03:25:01,493 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/6e82cea339894d5baceeab94ef78a028 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/B/6e82cea339894d5baceeab94ef78a028 2024-11-21T03:25:01,496 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8f5a2e2e20714316b36ea9daffa638d4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/8f5a2e2e20714316b36ea9daffa638d4 2024-11-21T03:25:01,501 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/90f3a5c9945e4089a9acf4b6720c4cb4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/C/90f3a5c9945e4089a9acf4b6720c4cb4 2024-11-21T03:25:01,505 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/recovered.edits/425.seqid to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c/recovered.edits/425.seqid 2024-11-21T03:25:01,505 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,505 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T03:25:01,506 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T03:25:01,507 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T03:25:01,511 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212082bd3550744f3095729924138e8b3b_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212082bd3550744f3095729924138e8b3b_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,512 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212cb211c9bd76486c867d8367311dec6c_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212cb211c9bd76486c867d8367311dec6c_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,514 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212fb66512d14f41eb952c55881e9f24d3_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212fb66512d14f41eb952c55881e9f24d3_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,515 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112130cb465ceee54209b3a81017e110a350_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112130cb465ceee54209b3a81017e110a350_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,516 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121442448a61c2648ecb65137bb5dd311cd_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121442448a61c2648ecb65137bb5dd311cd_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,518 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112168561e3bd6aa4619bba3886e53f0f208_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112168561e3bd6aa4619bba3886e53f0f208_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,519 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121709ff7178ec44b3d829aac3cdbab08b7_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121709ff7178ec44b3d829aac3cdbab08b7_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,520 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112176c59b59533448f6a7bad5739886e22c_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112176c59b59533448f6a7bad5739886e22c_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,521 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217992765103604d488b1f2170bfbceb12_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217992765103604d488b1f2170bfbceb12_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,522 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182f28fd4865749c1a69666569b09a51c_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182f28fd4865749c1a69666569b09a51c_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,524 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219894397ea69c4bd5a56a461d5785f33d_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219894397ea69c4bd5a56a461d5785f33d_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,525 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a492a031c7624bc28dbf198f05f527ed_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a492a031c7624bc28dbf198f05f527ed_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,526 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121aa1c36b263c24300b0834615389c0e34_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121aa1c36b263c24300b0834615389c0e34_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,527 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121aa62648b4b86461c9ef4271348489b9b_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121aa62648b4b86461c9ef4271348489b9b_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,528 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121abc9a9687a0643258c75bf1e98e9662e_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121abc9a9687a0643258c75bf1e98e9662e_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,530 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b258d9ea98d74e55ae71c00cae34112e_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b258d9ea98d74e55ae71c00cae34112e_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,531 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c524cca77be74f7c9c3c46623df3ef2b_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c524cca77be74f7c9c3c46623df3ef2b_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,532 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c544639a97814e5199bf08a144e0ce5c_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c544639a97814e5199bf08a144e0ce5c_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,534 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d8838a87de0e4be8b3bf81fd1fb6a8db_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d8838a87de0e4be8b3bf81fd1fb6a8db_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,535 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121da1a08f942f04b9c92c6da03ae3af7c1_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121da1a08f942f04b9c92c6da03ae3af7c1_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,536 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dc11afa9b7af450cb5f5aa9f2af4388f_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dc11afa9b7af450cb5f5aa9f2af4388f_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,537 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ec2d5967b9bd45cfab338b26385ab549_72b1610bcd54798702c9c5481e814b4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ec2d5967b9bd45cfab338b26385ab549_72b1610bcd54798702c9c5481e814b4c 2024-11-21T03:25:01,538 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T03:25:01,539 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:01,541 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T03:25:01,545 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T03:25:01,546 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:01,547 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T03:25:01,547 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732159501547"}]},"ts":"9223372036854775807"} 2024-11-21T03:25:01,549 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T03:25:01,549 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 72b1610bcd54798702c9c5481e814b4c, NAME => 'TestAcidGuarantees,,1732159470719.72b1610bcd54798702c9c5481e814b4c.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T03:25:01,549 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T03:25:01,549 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732159501549"}]},"ts":"9223372036854775807"} 2024-11-21T03:25:01,551 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T03:25:01,552 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:01,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-21T03:25:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T03:25:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T03:25:01,589 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:01,597 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 239), OpenFileDescriptor=457 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=997 (was 860) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3577 (was 2346) - AvailableMemoryMB LEAK? - 2024-11-21T03:25:01,608 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=997, ProcessCount=11, AvailableMemoryMB=3575 2024-11-21T03:25:01,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:25:01,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:25:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:01,612 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:25:01,613 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:01,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-21T03:25:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-21T03:25:01,616 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:25:01,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742069_1245 (size=959) 2024-11-21T03:25:01,626 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1 2024-11-21T03:25:01,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742070_1246 (size=53) 2024-11-21T03:25:01,648 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:25:01,648 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6b132942b031ffcc7f2f51c4c9e8428c, disabling compactions & flushes 2024-11-21T03:25:01,649 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:01,649 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:01,649 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. after waiting 0 ms 2024-11-21T03:25:01,649 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:01,649 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:01,649 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6b132942b031ffcc7f2f51c4c9e8428c: Waiting for close lock at 1732159501648Disabling compacts and flushes for region at 1732159501648Disabling writes for close at 1732159501649 (+1 ms)Writing region close event to WAL at 1732159501649Closed at 1732159501649 2024-11-21T03:25:01,650 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:25:01,650 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732159501650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732159501650"}]},"ts":"1732159501650"} 2024-11-21T03:25:01,652 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:25:01,653 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:25:01,654 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159501653"}]},"ts":"1732159501653"} 2024-11-21T03:25:01,655 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:25:01,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b132942b031ffcc7f2f51c4c9e8428c, ASSIGN}] 2024-11-21T03:25:01,657 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b132942b031ffcc7f2f51c4c9e8428c, ASSIGN 2024-11-21T03:25:01,658 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b132942b031ffcc7f2f51c4c9e8428c, ASSIGN; state=OFFLINE, location=b69e2a1ee061,45649,1732159441548; forceNewPlan=false, retain=false 2024-11-21T03:25:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-21T03:25:01,809 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=6b132942b031ffcc7f2f51c4c9e8428c, regionState=OPENING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:01,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b132942b031ffcc7f2f51c4c9e8428c, ASSIGN because future has completed 2024-11-21T03:25:01,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:25:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-21T03:25:01,969 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:01,969 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:25:01,969 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,969 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:25:01,969 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,969 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,972 INFO [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,974 INFO [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b132942b031ffcc7f2f51c4c9e8428c columnFamilyName A 2024-11-21T03:25:01,974 DEBUG [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:01,975 INFO [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] regionserver.HStore(327): Store=6b132942b031ffcc7f2f51c4c9e8428c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:25:01,975 INFO [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,976 INFO [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b132942b031ffcc7f2f51c4c9e8428c columnFamilyName B 2024-11-21T03:25:01,976 DEBUG [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:01,977 INFO [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] regionserver.HStore(327): Store=6b132942b031ffcc7f2f51c4c9e8428c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:25:01,977 INFO [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,979 INFO [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b132942b031ffcc7f2f51c4c9e8428c columnFamilyName C 2024-11-21T03:25:01,979 DEBUG [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:01,980 INFO [StoreOpener-6b132942b031ffcc7f2f51c4c9e8428c-1 {}] regionserver.HStore(327): Store=6b132942b031ffcc7f2f51c4c9e8428c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:25:01,980 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,981 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,981 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,982 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,982 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,982 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:25:01,984 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,986 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:25:01,986 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 6b132942b031ffcc7f2f51c4c9e8428c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72999113, jitterRate=0.08777154982089996}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:25:01,986 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:01,987 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 6b132942b031ffcc7f2f51c4c9e8428c: Running coprocessor pre-open hook at 1732159501969Writing region info on filesystem at 1732159501970 (+1 ms)Initializing all the Stores at 1732159501970Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159501970Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159501972 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159501972Cleaning up temporary data from old regions at 1732159501982 (+10 ms)Running coprocessor post-open hooks at 1732159501986 (+4 ms)Region opened successfully at 1732159501987 (+1 ms) 2024-11-21T03:25:01,988 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., pid=57, masterSystemTime=1732159501965 2024-11-21T03:25:01,991 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:01,991 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:01,991 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=6b132942b031ffcc7f2f51c4c9e8428c, regionState=OPEN, openSeqNum=2, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:01,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:25:01,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-21T03:25:01,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 in 182 msec 2024-11-21T03:25:01,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-21T03:25:01,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b132942b031ffcc7f2f51c4c9e8428c, ASSIGN in 339 msec 2024-11-21T03:25:01,998 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:25:01,998 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159501998"}]},"ts":"1732159501998"} 2024-11-21T03:25:02,000 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:25:02,001 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:25:02,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 391 msec 2024-11-21T03:25:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-21T03:25:02,238 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:02,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,243 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519347e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,245 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67378625, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,246 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,249 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:25:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is b69e2a1ee061,42023,1732159440464 2024-11-21T03:25:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@af4868c 2024-11-21T03:25:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:25:02,251 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:25:02,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,256 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c141, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,258 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4753476a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,260 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1070b666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,262 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76102e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,264 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31296bf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,266 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c66c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,268 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5769f4a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,270 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4336e3f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,273 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5675b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,275 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20eee107, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,279 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36d751d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,280 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b683f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,286 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428cc673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,289 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5053a1c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,293 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a46afbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,295 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47742f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,298 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6f6c11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,300 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f04dd03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:02,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:02,303 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6a704e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:02,305 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:02,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e47485e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:02,310 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,310 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,312 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,314 DEBUG [Thread-1158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,314 DEBUG [Thread-1160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,315 DEBUG [Thread-1160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,315 DEBUG [Thread-1160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,316 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,316 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:02,316 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,320 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:25:02,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:25:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-21T03:25:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:25:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:25:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:25:02,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,323 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:02,323 DEBUG [Thread-1156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,324 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:02,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:02,324 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:25:02,325 DEBUG [Thread-1143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,325 DEBUG [Thread-1143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:25:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:25:02,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,331 DEBUG [Thread-1156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:25:02,346 DEBUG [Thread-1156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:25:02,351 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:02,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:25:02,353 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:02,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:25:02,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/f719985fe60846628dab220cd2bf718b is 50, key is test_row_0/A:col10/1732159502335/Put/seqid=0 2024-11-21T03:25:02,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159562399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159562400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159562401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159562403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159562409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:25:02,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742071_1247 (size=12001) 2024-11-21T03:25:02,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T03:25:02,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:02,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:02,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:02,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159562518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159562519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159562519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159562519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159562527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:25:02,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T03:25:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:02,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159562728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159562728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159562728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159562728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159562738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:02,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T03:25:02,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:02,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:02,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/f719985fe60846628dab220cd2bf718b 2024-11-21T03:25:02,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/9e0a4ad91f284bfc8837cbe38b69501f is 50, key is test_row_0/B:col10/1732159502335/Put/seqid=0 2024-11-21T03:25:02,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742072_1248 (size=12001) 2024-11-21T03:25:02,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/9e0a4ad91f284bfc8837cbe38b69501f 2024-11-21T03:25:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:25:02,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T03:25:02,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:02,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:02,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:02,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:02,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/0a966dcd8580400aadef2656b0ea064d is 50, key is test_row_0/C:col10/1732159502335/Put/seqid=0 2024-11-21T03:25:02,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742073_1249 (size=12001) 2024-11-21T03:25:02,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/0a966dcd8580400aadef2656b0ea064d 2024-11-21T03:25:03,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/f719985fe60846628dab220cd2bf718b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f719985fe60846628dab220cd2bf718b 2024-11-21T03:25:03,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159563038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159563038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159563038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159563038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f719985fe60846628dab220cd2bf718b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-21T03:25:03,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/9e0a4ad91f284bfc8837cbe38b69501f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/9e0a4ad91f284bfc8837cbe38b69501f 2024-11-21T03:25:03,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159563049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/9e0a4ad91f284bfc8837cbe38b69501f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-21T03:25:03,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/0a966dcd8580400aadef2656b0ea064d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a966dcd8580400aadef2656b0ea064d 2024-11-21T03:25:03,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a966dcd8580400aadef2656b0ea064d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-21T03:25:03,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6b132942b031ffcc7f2f51c4c9e8428c in 734ms, sequenceid=16, compaction requested=false 2024-11-21T03:25:03,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:03,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T03:25:03,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:03,128 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T03:25:03,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/c34959e2356d4ea598755384b7f53738 is 50, key is test_row_0/A:col10/1732159502406/Put/seqid=0 2024-11-21T03:25:03,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742074_1250 (size=12001) 2024-11-21T03:25:03,191 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/c34959e2356d4ea598755384b7f53738 2024-11-21T03:25:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/140f2d8f35734cfcb06a94a3c8d15a25 is 50, key is test_row_0/B:col10/1732159502406/Put/seqid=0 2024-11-21T03:25:03,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742075_1251 (size=12001) 2024-11-21T03:25:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:25:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:03,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:03,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159563622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159563623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159563624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159563625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159563628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,636 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/140f2d8f35734cfcb06a94a3c8d15a25 2024-11-21T03:25:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/e485fe82194c4763ade81a50a75a187f is 50, key is test_row_0/C:col10/1732159502406/Put/seqid=0 2024-11-21T03:25:03,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742076_1252 (size=12001) 2024-11-21T03:25:03,701 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/e485fe82194c4763ade81a50a75a187f 2024-11-21T03:25:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/c34959e2356d4ea598755384b7f53738 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c34959e2356d4ea598755384b7f53738 2024-11-21T03:25:03,728 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c34959e2356d4ea598755384b7f53738, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:25:03,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/140f2d8f35734cfcb06a94a3c8d15a25 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/140f2d8f35734cfcb06a94a3c8d15a25 2024-11-21T03:25:03,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159563738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159563738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159563738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159563738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,741 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/140f2d8f35734cfcb06a94a3c8d15a25, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:25:03,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/e485fe82194c4763ade81a50a75a187f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e485fe82194c4763ade81a50a75a187f 2024-11-21T03:25:03,747 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e485fe82194c4763ade81a50a75a187f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:25:03,748 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6b132942b031ffcc7f2f51c4c9e8428c in 620ms, sequenceid=37, compaction requested=false 2024-11-21T03:25:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-21T03:25:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-21T03:25:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:03,751 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:03,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T03:25:03,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-21T03:25:03,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4250 sec 2024-11-21T03:25:03,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.4420 sec 2024-11-21T03:25:03,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/8ab32959f4f7420d9a57ab7822aeccdb is 50, key is test_row_0/A:col10/1732159503625/Put/seqid=0 2024-11-21T03:25:03,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742077_1253 (size=12001) 2024-11-21T03:25:03,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/8ab32959f4f7420d9a57ab7822aeccdb 2024-11-21T03:25:03,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/f6ab0304a1c34fe38d19f2d0348a4980 is 50, key is test_row_0/B:col10/1732159503625/Put/seqid=0 2024-11-21T03:25:03,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742078_1254 (size=12001) 2024-11-21T03:25:03,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/f6ab0304a1c34fe38d19f2d0348a4980 2024-11-21T03:25:03,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/e970478390e44505aa422980350f0f0d is 50, key is test_row_0/C:col10/1732159503625/Put/seqid=0 2024-11-21T03:25:03,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159563873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742079_1255 (size=12001) 2024-11-21T03:25:03,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/e970478390e44505aa422980350f0f0d 2024-11-21T03:25:03,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/8ab32959f4f7420d9a57ab7822aeccdb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8ab32959f4f7420d9a57ab7822aeccdb 2024-11-21T03:25:03,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8ab32959f4f7420d9a57ab7822aeccdb, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T03:25:03,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/f6ab0304a1c34fe38d19f2d0348a4980 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f6ab0304a1c34fe38d19f2d0348a4980 2024-11-21T03:25:03,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f6ab0304a1c34fe38d19f2d0348a4980, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T03:25:03,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/e970478390e44505aa422980350f0f0d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e970478390e44505aa422980350f0f0d 2024-11-21T03:25:03,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e970478390e44505aa422980350f0f0d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T03:25:03,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6b132942b031ffcc7f2f51c4c9e8428c in 175ms, sequenceid=53, compaction requested=true 2024-11-21T03:25:03,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:03,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:03,927 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:03,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:03,927 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:03,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:03,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:03,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:03,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:03,928 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:03,929 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:03,929 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:03,929 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/9e0a4ad91f284bfc8837cbe38b69501f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/140f2d8f35734cfcb06a94a3c8d15a25, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f6ab0304a1c34fe38d19f2d0348a4980] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=35.2 K 2024-11-21T03:25:03,929 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:03,929 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:03,929 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:03,929 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f719985fe60846628dab220cd2bf718b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c34959e2356d4ea598755384b7f53738, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8ab32959f4f7420d9a57ab7822aeccdb] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=35.2 K 2024-11-21T03:25:03,930 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f719985fe60846628dab220cd2bf718b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732159502335 2024-11-21T03:25:03,930 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e0a4ad91f284bfc8837cbe38b69501f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732159502335 2024-11-21T03:25:03,931 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 140f2d8f35734cfcb06a94a3c8d15a25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732159502398 2024-11-21T03:25:03,931 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c34959e2356d4ea598755384b7f53738, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732159502398 2024-11-21T03:25:03,931 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting f6ab0304a1c34fe38d19f2d0348a4980, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732159503606 2024-11-21T03:25:03,932 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab32959f4f7420d9a57ab7822aeccdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732159503606 2024-11-21T03:25:03,951 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:03,952 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/e122ea5a791647fca67fc16feb8dab13 is 50, key is test_row_0/A:col10/1732159503625/Put/seqid=0 2024-11-21T03:25:03,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:03,955 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:03,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T03:25:03,963 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:03,964 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/f4eff044f17d4d3aa8b1cbaa19bc4b35 is 50, key is test_row_0/B:col10/1732159503625/Put/seqid=0 2024-11-21T03:25:03,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/d79edddc43f54780aa64241e9148cf19 is 50, key is test_row_0/A:col10/1732159503955/Put/seqid=0 2024-11-21T03:25:03,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159563983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159563984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159563988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159563989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159563990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742080_1256 (size=12104) 2024-11-21T03:25:04,023 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/e122ea5a791647fca67fc16feb8dab13 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e122ea5a791647fca67fc16feb8dab13 2024-11-21T03:25:04,033 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into e122ea5a791647fca67fc16feb8dab13(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:04,033 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:04,033 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=13, startTime=1732159503926; duration=0sec 2024-11-21T03:25:04,033 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:04,033 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:04,033 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:04,035 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:04,035 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:04,035 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:04,035 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a966dcd8580400aadef2656b0ea064d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e485fe82194c4763ade81a50a75a187f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e970478390e44505aa422980350f0f0d] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=35.2 K 2024-11-21T03:25:04,036 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a966dcd8580400aadef2656b0ea064d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732159502335 2024-11-21T03:25:04,036 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e485fe82194c4763ade81a50a75a187f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732159502398 2024-11-21T03:25:04,037 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e970478390e44505aa422980350f0f0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732159503606 2024-11-21T03:25:04,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742081_1257 (size=12104) 2024-11-21T03:25:04,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742082_1258 (size=19021) 2024-11-21T03:25:04,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/d79edddc43f54780aa64241e9148cf19 2024-11-21T03:25:04,063 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/f4eff044f17d4d3aa8b1cbaa19bc4b35 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f4eff044f17d4d3aa8b1cbaa19bc4b35 2024-11-21T03:25:04,079 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:04,080 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/78b80e5625994796a53cb9b6db5c9831 is 50, key is test_row_0/C:col10/1732159503625/Put/seqid=0 2024-11-21T03:25:04,085 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into f4eff044f17d4d3aa8b1cbaa19bc4b35(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:04,085 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:04,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/e7f3df9615e4414e9c81b90c9f86b00b is 50, key is test_row_0/B:col10/1732159503955/Put/seqid=0 2024-11-21T03:25:04,085 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=13, startTime=1732159503927; duration=0sec 2024-11-21T03:25:04,085 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:04,085 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:04,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159564098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159564100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159564108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159564108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742084_1260 (size=12001) 2024-11-21T03:25:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742083_1259 (size=12104) 2024-11-21T03:25:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/e7f3df9615e4414e9c81b90c9f86b00b 2024-11-21T03:25:04,152 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/78b80e5625994796a53cb9b6db5c9831 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/78b80e5625994796a53cb9b6db5c9831 2024-11-21T03:25:04,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/ed6e8937c0ae4fdfbf206142345c97d6 is 50, key is test_row_0/C:col10/1732159503955/Put/seqid=0 2024-11-21T03:25:04,163 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 78b80e5625994796a53cb9b6db5c9831(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:04,163 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:04,163 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=13, startTime=1732159503927; duration=0sec 2024-11-21T03:25:04,163 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:04,163 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159564198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742085_1261 (size=12001) 2024-11-21T03:25:04,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/ed6e8937c0ae4fdfbf206142345c97d6 2024-11-21T03:25:04,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/d79edddc43f54780aa64241e9148cf19 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d79edddc43f54780aa64241e9148cf19 2024-11-21T03:25:04,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d79edddc43f54780aa64241e9148cf19, entries=300, sequenceid=76, filesize=18.6 K 2024-11-21T03:25:04,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/e7f3df9615e4414e9c81b90c9f86b00b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e7f3df9615e4414e9c81b90c9f86b00b 2024-11-21T03:25:04,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e7f3df9615e4414e9c81b90c9f86b00b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-21T03:25:04,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/ed6e8937c0ae4fdfbf206142345c97d6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ed6e8937c0ae4fdfbf206142345c97d6 2024-11-21T03:25:04,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ed6e8937c0ae4fdfbf206142345c97d6, entries=150, sequenceid=76, filesize=11.7 K 2024-11-21T03:25:04,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 6b132942b031ffcc7f2f51c4c9e8428c in 316ms, sequenceid=76, compaction requested=false 2024-11-21T03:25:04,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:04,311 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:04,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T03:25:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:04,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/77c85c605f8048859e44bdbf3fdff50f is 50, key is test_row_0/A:col10/1732159503987/Put/seqid=0 2024-11-21T03:25:04,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742086_1262 (size=14341) 2024-11-21T03:25:04,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159564368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159564374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159564374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159564375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:25:04,470 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:04,470 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-21T03:25:04,472 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:25:04,473 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:04,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:04,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159564488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159564488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159564490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159564497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159564507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:25:04,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:25:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:04,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:04,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:04,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159564701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159564701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159564704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159564708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:04,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/77c85c605f8048859e44bdbf3fdff50f 2024-11-21T03:25:04,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/113af1cc13834e648e1ed4c96b91ccd3 is 50, key is test_row_0/B:col10/1732159503987/Put/seqid=0 2024-11-21T03:25:04,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:25:04,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:04,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:04,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:04,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:04,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:25:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:04,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742087_1263 (size=12001) 2024-11-21T03:25:04,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/113af1cc13834e648e1ed4c96b91ccd3 2024-11-21T03:25:04,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/2b28104ee4bc4f57b37541895fc44866 is 50, key is test_row_0/C:col10/1732159503987/Put/seqid=0 2024-11-21T03:25:04,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742088_1264 (size=12001) 2024-11-21T03:25:04,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/2b28104ee4bc4f57b37541895fc44866 2024-11-21T03:25:04,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:25:04,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:04,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:04,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:04,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:04,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:04,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/77c85c605f8048859e44bdbf3fdff50f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/77c85c605f8048859e44bdbf3fdff50f 2024-11-21T03:25:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:04,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/77c85c605f8048859e44bdbf3fdff50f, entries=200, sequenceid=94, filesize=14.0 K 2024-11-21T03:25:04,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/113af1cc13834e648e1ed4c96b91ccd3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/113af1cc13834e648e1ed4c96b91ccd3 2024-11-21T03:25:04,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/113af1cc13834e648e1ed4c96b91ccd3, entries=150, sequenceid=94, filesize=11.7 K 2024-11-21T03:25:04,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/2b28104ee4bc4f57b37541895fc44866 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b28104ee4bc4f57b37541895fc44866 2024-11-21T03:25:04,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b28104ee4bc4f57b37541895fc44866, entries=150, sequenceid=94, filesize=11.7 K 2024-11-21T03:25:05,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=134.18 KB/137400 for 6b132942b031ffcc7f2f51c4c9e8428c in 689ms, sequenceid=94, compaction requested=true 2024-11-21T03:25:05,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:05,001 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:05,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:05,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:05,001 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:05,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:05,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:05,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:05,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:05,003 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45466 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:05,004 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:05,004 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,004 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e122ea5a791647fca67fc16feb8dab13, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d79edddc43f54780aa64241e9148cf19, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/77c85c605f8048859e44bdbf3fdff50f] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=44.4 K 2024-11-21T03:25:05,004 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:05,005 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:05,005 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,005 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f4eff044f17d4d3aa8b1cbaa19bc4b35, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e7f3df9615e4414e9c81b90c9f86b00b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/113af1cc13834e648e1ed4c96b91ccd3] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=35.3 K 2024-11-21T03:25:05,005 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e122ea5a791647fca67fc16feb8dab13, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732159503606 2024-11-21T03:25:05,007 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting f4eff044f17d4d3aa8b1cbaa19bc4b35, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732159503606 2024-11-21T03:25:05,008 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d79edddc43f54780aa64241e9148cf19, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732159503853 2024-11-21T03:25:05,009 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e7f3df9615e4414e9c81b90c9f86b00b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732159503864 2024-11-21T03:25:05,009 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77c85c605f8048859e44bdbf3fdff50f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732159503979 2024-11-21T03:25:05,010 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 113af1cc13834e648e1ed4c96b91ccd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732159503979 2024-11-21T03:25:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:05,026 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:05,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:25:05,036 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:05,037 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/c1c318920c6d41caa3e68b6174a1506f is 50, key is test_row_0/B:col10/1732159503987/Put/seqid=0 2024-11-21T03:25:05,048 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#217 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:05,048 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/5dd496c7f2064e2a9fbd32dbea8561be is 50, key is test_row_0/A:col10/1732159503987/Put/seqid=0 2024-11-21T03:25:05,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/837e0ac303e34f4c918f9287c1435faa is 50, key is test_row_0/A:col10/1732159504369/Put/seqid=0 2024-11-21T03:25:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159565052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159565051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159565058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159565061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159565063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742089_1265 (size=12207) 2024-11-21T03:25:05,093 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/c1c318920c6d41caa3e68b6174a1506f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c1c318920c6d41caa3e68b6174a1506f 2024-11-21T03:25:05,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742090_1266 (size=12207) 2024-11-21T03:25:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:25:05,101 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into c1c318920c6d41caa3e68b6174a1506f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:05,101 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:05,101 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=13, startTime=1732159505001; duration=0sec 2024-11-21T03:25:05,102 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:05,102 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:05,102 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:05,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:25:05,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742091_1267 (size=12001) 2024-11-21T03:25:05,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:05,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,109 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:05,109 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:05,109 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,109 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/78b80e5625994796a53cb9b6db5c9831, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ed6e8937c0ae4fdfbf206142345c97d6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b28104ee4bc4f57b37541895fc44866] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=35.3 K 2024-11-21T03:25:05,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/837e0ac303e34f4c918f9287c1435faa 2024-11-21T03:25:05,119 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 78b80e5625994796a53cb9b6db5c9831, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732159503606 2024-11-21T03:25:05,120 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/5dd496c7f2064e2a9fbd32dbea8561be as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5dd496c7f2064e2a9fbd32dbea8561be 2024-11-21T03:25:05,120 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting ed6e8937c0ae4fdfbf206142345c97d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732159503864 2024-11-21T03:25:05,123 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b28104ee4bc4f57b37541895fc44866, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732159503979 2024-11-21T03:25:05,162 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into 5dd496c7f2064e2a9fbd32dbea8561be(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:05,162 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:05,162 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=13, startTime=1732159505000; duration=0sec 2024-11-21T03:25:05,163 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:05,163 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:05,164 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:05,165 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/69166415d8404226beff221db57c0a5c is 50, key is test_row_0/C:col10/1732159503987/Put/seqid=0 2024-11-21T03:25:05,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/cca0641a0db8432b91a634bc66f14289 is 50, key is test_row_0/B:col10/1732159504369/Put/seqid=0 2024-11-21T03:25:05,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159565178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159565179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159565180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159565181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159565190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742092_1268 (size=12207) 2024-11-21T03:25:05,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742093_1269 (size=12001) 2024-11-21T03:25:05,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/cca0641a0db8432b91a634bc66f14289 2024-11-21T03:25:05,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:25:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/71f0f747178d49dcaf0f69ab563ca8a3 is 50, key is test_row_0/C:col10/1732159504369/Put/seqid=0 2024-11-21T03:25:05,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742094_1270 (size=12001) 2024-11-21T03:25:05,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159565397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159565397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159565398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159565400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159565408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:25:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:25:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:25:05,651 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/69166415d8404226beff221db57c0a5c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/69166415d8404226beff221db57c0a5c 2024-11-21T03:25:05,664 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 69166415d8404226beff221db57c0a5c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:05,664 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:05,664 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=13, startTime=1732159505001; duration=0sec 2024-11-21T03:25:05,664 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:05,664 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:05,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159565708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159565718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159565728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159565728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:25:05,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159565738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:05,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/71f0f747178d49dcaf0f69ab563ca8a3 2024-11-21T03:25:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:05,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:05,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/837e0ac303e34f4c918f9287c1435faa as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/837e0ac303e34f4c918f9287c1435faa 2024-11-21T03:25:05,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/837e0ac303e34f4c918f9287c1435faa, entries=150, sequenceid=120, filesize=11.7 K 2024-11-21T03:25:05,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/cca0641a0db8432b91a634bc66f14289 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/cca0641a0db8432b91a634bc66f14289 2024-11-21T03:25:05,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/cca0641a0db8432b91a634bc66f14289, entries=150, sequenceid=120, filesize=11.7 K 2024-11-21T03:25:05,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/71f0f747178d49dcaf0f69ab563ca8a3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/71f0f747178d49dcaf0f69ab563ca8a3 2024-11-21T03:25:05,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/71f0f747178d49dcaf0f69ab563ca8a3, entries=150, sequenceid=120, filesize=11.7 K 2024-11-21T03:25:05,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6b132942b031ffcc7f2f51c4c9e8428c in 759ms, sequenceid=120, compaction requested=false 2024-11-21T03:25:05,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:05,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:25:05,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:05,912 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T03:25:05,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/65fb1739bcd74206a795b3c2a5538967 is 50, key is test_row_0/A:col10/1732159505049/Put/seqid=0 2024-11-21T03:25:05,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742095_1271 (size=12051) 2024-11-21T03:25:06,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:06,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159566270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159566271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159566272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159566276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159566277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,294 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:25:06,381 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/65fb1739bcd74206a795b3c2a5538967 2024-11-21T03:25:06,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159566388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159566388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159566388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159566388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159566388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/8ad66e183db1471898f05da76debc4b6 is 50, key is test_row_0/B:col10/1732159505049/Put/seqid=0 2024-11-21T03:25:06,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742096_1272 (size=12051) 2024-11-21T03:25:06,440 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/8ad66e183db1471898f05da76debc4b6 2024-11-21T03:25:06,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/861f3c1c9354456ab63e9fb6363219a7 is 50, key is test_row_0/C:col10/1732159505049/Put/seqid=0 2024-11-21T03:25:06,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742097_1273 (size=12051) 2024-11-21T03:25:06,485 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/861f3c1c9354456ab63e9fb6363219a7 2024-11-21T03:25:06,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/65fb1739bcd74206a795b3c2a5538967 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/65fb1739bcd74206a795b3c2a5538967 2024-11-21T03:25:06,501 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/65fb1739bcd74206a795b3c2a5538967, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T03:25:06,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/8ad66e183db1471898f05da76debc4b6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8ad66e183db1471898f05da76debc4b6 2024-11-21T03:25:06,526 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8ad66e183db1471898f05da76debc4b6, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T03:25:06,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/861f3c1c9354456ab63e9fb6363219a7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/861f3c1c9354456ab63e9fb6363219a7 2024-11-21T03:25:06,548 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/861f3c1c9354456ab63e9fb6363219a7, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T03:25:06,549 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 6b132942b031ffcc7f2f51c4c9e8428c in 637ms, sequenceid=133, compaction requested=true 2024-11-21T03:25:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:06,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-21T03:25:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-21T03:25:06,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-21T03:25:06,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0990 sec 2024-11-21T03:25:06,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.1070 sec 2024-11-21T03:25:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:06,605 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:06,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-21T03:25:06,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159566609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159566611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:25:06,619 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:06,620 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:06,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159566615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159566615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-21T03:25:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159566616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:25:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,624 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:06,626 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:06,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:06,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/788bfe861d1946bfbcfca72df6035754 is 50, key is test_row_0/A:col10/1732159506269/Put/seqid=0 2024-11-21T03:25:06,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742098_1274 (size=12151) 2024-11-21T03:25:06,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/788bfe861d1946bfbcfca72df6035754 2024-11-21T03:25:06,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/dea1d3c9d4f646e494865f5678ebcc51 is 50, key is test_row_0/B:col10/1732159506269/Put/seqid=0 2024-11-21T03:25:06,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742099_1275 (size=12151) 2024-11-21T03:25:06,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/dea1d3c9d4f646e494865f5678ebcc51 2024-11-21T03:25:06,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159566717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159566718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:25:06,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159566728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159566728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159566729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:06,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:06,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/54fbe3b158c64cfb8e513cefce82c95e is 50, key is test_row_0/C:col10/1732159506269/Put/seqid=0 2024-11-21T03:25:06,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742100_1276 (size=12151) 2024-11-21T03:25:06,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/54fbe3b158c64cfb8e513cefce82c95e 2024-11-21T03:25:06,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/788bfe861d1946bfbcfca72df6035754 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/788bfe861d1946bfbcfca72df6035754 2024-11-21T03:25:06,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/788bfe861d1946bfbcfca72df6035754, entries=150, sequenceid=163, filesize=11.9 K 2024-11-21T03:25:06,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/dea1d3c9d4f646e494865f5678ebcc51 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dea1d3c9d4f646e494865f5678ebcc51 2024-11-21T03:25:06,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dea1d3c9d4f646e494865f5678ebcc51, entries=150, sequenceid=163, filesize=11.9 K 2024-11-21T03:25:06,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T03:25:06,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:06,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:06,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:06,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:06,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:06,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/54fbe3b158c64cfb8e513cefce82c95e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/54fbe3b158c64cfb8e513cefce82c95e 2024-11-21T03:25:06,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/54fbe3b158c64cfb8e513cefce82c95e, entries=150, sequenceid=163, filesize=11.9 K 2024-11-21T03:25:06,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=33.54 KB/34350 for 6b132942b031ffcc7f2f51c4c9e8428c in 198ms, sequenceid=163, compaction requested=true 2024-11-21T03:25:06,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:06,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:06,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:06,804 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:06,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:06,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:06,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:06,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:25:06,804 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:06,806 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:06,807 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:06,807 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:06,807 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c1c318920c6d41caa3e68b6174a1506f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/cca0641a0db8432b91a634bc66f14289, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8ad66e183db1471898f05da76debc4b6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dea1d3c9d4f646e494865f5678ebcc51] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=47.3 K 2024-11-21T03:25:06,807 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:06,807 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:06,807 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:06,807 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5dd496c7f2064e2a9fbd32dbea8561be, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/837e0ac303e34f4c918f9287c1435faa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/65fb1739bcd74206a795b3c2a5538967, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/788bfe861d1946bfbcfca72df6035754] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=47.3 K 2024-11-21T03:25:06,808 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting c1c318920c6d41caa3e68b6174a1506f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732159503979 2024-11-21T03:25:06,808 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dd496c7f2064e2a9fbd32dbea8561be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732159503979 2024-11-21T03:25:06,808 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting cca0641a0db8432b91a634bc66f14289, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732159504369 2024-11-21T03:25:06,809 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 837e0ac303e34f4c918f9287c1435faa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732159504369 2024-11-21T03:25:06,809 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65fb1739bcd74206a795b3c2a5538967, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159505049 2024-11-21T03:25:06,810 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 788bfe861d1946bfbcfca72df6035754, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732159506268 2024-11-21T03:25:06,811 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad66e183db1471898f05da76debc4b6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159505049 2024-11-21T03:25:06,811 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting dea1d3c9d4f646e494865f5678ebcc51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732159506268 2024-11-21T03:25:06,839 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:06,840 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/c555bfcfefdf4f87bec932644e979062 is 50, key is test_row_0/A:col10/1732159506269/Put/seqid=0 2024-11-21T03:25:06,846 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:06,847 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/7c83412a47354a15af78da939c195e5a is 50, key is test_row_0/B:col10/1732159506269/Put/seqid=0 2024-11-21T03:25:06,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742101_1277 (size=12493) 2024-11-21T03:25:06,888 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/c555bfcfefdf4f87bec932644e979062 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c555bfcfefdf4f87bec932644e979062 2024-11-21T03:25:06,895 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into c555bfcfefdf4f87bec932644e979062(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:06,895 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:06,895 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=12, startTime=1732159506804; duration=0sec 2024-11-21T03:25:06,895 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:06,895 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:06,895 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:06,899 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:06,899 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:06,899 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:06,900 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/69166415d8404226beff221db57c0a5c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/71f0f747178d49dcaf0f69ab563ca8a3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/861f3c1c9354456ab63e9fb6363219a7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/54fbe3b158c64cfb8e513cefce82c95e] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=47.3 K 2024-11-21T03:25:06,900 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69166415d8404226beff221db57c0a5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732159503979 2024-11-21T03:25:06,901 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f0f747178d49dcaf0f69ab563ca8a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732159504369 2024-11-21T03:25:06,901 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 861f3c1c9354456ab63e9fb6363219a7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159505049 2024-11-21T03:25:06,903 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54fbe3b158c64cfb8e513cefce82c95e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732159506268 2024-11-21T03:25:06,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742102_1278 (size=12493) 2024-11-21T03:25:06,929 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/7c83412a47354a15af78da939c195e5a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7c83412a47354a15af78da939c195e5a 2024-11-21T03:25:06,930 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:06,930 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/25b6a047a3474f08b49dd76f4c54dea7 is 50, key is test_row_0/C:col10/1732159506269/Put/seqid=0 2024-11-21T03:25:06,936 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into 7c83412a47354a15af78da939c195e5a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:06,936 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:06,936 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=12, startTime=1732159506804; duration=0sec 2024-11-21T03:25:06,937 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:06,937 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:25:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:06,946 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:06,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:25:06,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T03:25:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:06,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:06,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:06,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:06,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:06,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:06,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742103_1279 (size=12493) 2024-11-21T03:25:06,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/867bab1cad104d0da6580ec5270e7b64 is 50, key is test_row_0/A:col10/1732159506944/Put/seqid=0 2024-11-21T03:25:06,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742104_1280 (size=12151) 2024-11-21T03:25:06,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/867bab1cad104d0da6580ec5270e7b64 2024-11-21T03:25:06,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/4e8d5e74170942d0848606bfd05dff6f is 50, key is test_row_0/B:col10/1732159506944/Put/seqid=0 2024-11-21T03:25:07,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159566996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159566996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159566999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159567005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159567004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742105_1281 (size=12151) 2024-11-21T03:25:07,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T03:25:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159567108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159567118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159567118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159567120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159567127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:25:07,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T03:25:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159567319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159567332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159567332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159567338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159567338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,361 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/25b6a047a3474f08b49dd76f4c54dea7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/25b6a047a3474f08b49dd76f4c54dea7 2024-11-21T03:25:07,376 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 25b6a047a3474f08b49dd76f4c54dea7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:07,376 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:07,376 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=12, startTime=1732159506804; duration=0sec 2024-11-21T03:25:07,377 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:07,377 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:07,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T03:25:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/4e8d5e74170942d0848606bfd05dff6f 2024-11-21T03:25:07,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/f0f0a05571e549d298c7bd6b8035ed75 is 50, key is test_row_0/C:col10/1732159506944/Put/seqid=0 2024-11-21T03:25:07,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742106_1282 (size=12151) 2024-11-21T03:25:07,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T03:25:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159567628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159567638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159567647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159567648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159567648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:07,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:07,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:07,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T03:25:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:25:07,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T03:25:07,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:07,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:07,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/f0f0a05571e549d298c7bd6b8035ed75 2024-11-21T03:25:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/867bab1cad104d0da6580ec5270e7b64 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/867bab1cad104d0da6580ec5270e7b64 2024-11-21T03:25:07,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/867bab1cad104d0da6580ec5270e7b64, entries=150, sequenceid=177, filesize=11.9 K 2024-11-21T03:25:07,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/4e8d5e74170942d0848606bfd05dff6f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/4e8d5e74170942d0848606bfd05dff6f 2024-11-21T03:25:07,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/4e8d5e74170942d0848606bfd05dff6f, entries=150, sequenceid=177, filesize=11.9 K 2024-11-21T03:25:07,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/f0f0a05571e549d298c7bd6b8035ed75 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0f0a05571e549d298c7bd6b8035ed75 2024-11-21T03:25:07,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0f0a05571e549d298c7bd6b8035ed75, entries=150, sequenceid=177, filesize=11.9 K 2024-11-21T03:25:07,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6b132942b031ffcc7f2f51c4c9e8428c in 1019ms, sequenceid=177, compaction requested=false 2024-11-21T03:25:07,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:08,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-21T03:25:08,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:08,049 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:25:08,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/f52ecaa049fd409490565c5887e9d60e is 50, key is test_row_0/A:col10/1732159506983/Put/seqid=0 2024-11-21T03:25:08,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742107_1283 (size=12151) 2024-11-21T03:25:08,089 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/f52ecaa049fd409490565c5887e9d60e 2024-11-21T03:25:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/2afc9bfd7e5343a0aba161ab5b50a47e is 50, key is test_row_0/B:col10/1732159506983/Put/seqid=0 2024-11-21T03:25:08,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742108_1284 (size=12151) 2024-11-21T03:25:08,126 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/2afc9bfd7e5343a0aba161ab5b50a47e 2024-11-21T03:25:08,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/995c00fb7b1c42a0af737c6ca7aaa29d is 50, key is test_row_0/C:col10/1732159506983/Put/seqid=0 2024-11-21T03:25:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:08,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:08,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742109_1285 (size=12151) 2024-11-21T03:25:08,175 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/995c00fb7b1c42a0af737c6ca7aaa29d 2024-11-21T03:25:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/f52ecaa049fd409490565c5887e9d60e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f52ecaa049fd409490565c5887e9d60e 2024-11-21T03:25:08,201 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f52ecaa049fd409490565c5887e9d60e, entries=150, sequenceid=202, filesize=11.9 K 2024-11-21T03:25:08,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/2afc9bfd7e5343a0aba161ab5b50a47e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/2afc9bfd7e5343a0aba161ab5b50a47e 2024-11-21T03:25:08,207 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/2afc9bfd7e5343a0aba161ab5b50a47e, entries=150, sequenceid=202, filesize=11.9 K 2024-11-21T03:25:08,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/995c00fb7b1c42a0af737c6ca7aaa29d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/995c00fb7b1c42a0af737c6ca7aaa29d 2024-11-21T03:25:08,220 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/995c00fb7b1c42a0af737c6ca7aaa29d, entries=150, sequenceid=202, filesize=11.9 K 2024-11-21T03:25:08,223 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=53.67 KB/54960 for 6b132942b031ffcc7f2f51c4c9e8428c in 174ms, sequenceid=202, compaction requested=true 2024-11-21T03:25:08,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:08,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:08,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-21T03:25:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:08,228 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:08,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:25:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-21T03:25:08,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/0cac5dc274724286a52a4286a6f929f4 is 50, key is test_row_0/A:col10/1732159508226/Put/seqid=0 2024-11-21T03:25:08,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-21T03:25:08,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6120 sec 2024-11-21T03:25:08,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.6220 sec 2024-11-21T03:25:08,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742110_1286 (size=14541) 2024-11-21T03:25:08,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159568281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159568284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159568287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159568287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159568288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159568398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159568398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159568408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159568408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159568408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159568608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159568608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159568617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159568617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159568618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/0cac5dc274724286a52a4286a6f929f4 2024-11-21T03:25:08,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/85d5635031e94c7387e073088de6495f is 50, key is test_row_0/B:col10/1732159508226/Put/seqid=0 2024-11-21T03:25:08,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742111_1287 (size=12151) 2024-11-21T03:25:08,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:25:08,768 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:08,769 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:08,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-21T03:25:08,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T03:25:08,780 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:08,782 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:08,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T03:25:08,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159568917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159568917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159568928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159568928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159568928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:08,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T03:25:08,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:08,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:08,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:08,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:08,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T03:25:09,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T03:25:09,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:09,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/85d5635031e94c7387e073088de6495f 2024-11-21T03:25:09,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/65860daa1a9e4dfca52f127ccd18a0e8 is 50, key is test_row_0/C:col10/1732159508226/Put/seqid=0 2024-11-21T03:25:09,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742112_1288 (size=12151) 2024-11-21T03:25:09,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T03:25:09,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T03:25:09,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T03:25:09,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:09,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159569427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159569437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159569437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159569438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159569438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:09,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T03:25:09,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/65860daa1a9e4dfca52f127ccd18a0e8 2024-11-21T03:25:09,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:09,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/0cac5dc274724286a52a4286a6f929f4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/0cac5dc274724286a52a4286a6f929f4 2024-11-21T03:25:09,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/0cac5dc274724286a52a4286a6f929f4, entries=200, sequenceid=214, filesize=14.2 K 2024-11-21T03:25:09,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/85d5635031e94c7387e073088de6495f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/85d5635031e94c7387e073088de6495f 2024-11-21T03:25:09,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/85d5635031e94c7387e073088de6495f, entries=150, sequenceid=214, filesize=11.9 K 2024-11-21T03:25:09,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/65860daa1a9e4dfca52f127ccd18a0e8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65860daa1a9e4dfca52f127ccd18a0e8 2024-11-21T03:25:09,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65860daa1a9e4dfca52f127ccd18a0e8, entries=150, sequenceid=214, filesize=11.9 K 2024-11-21T03:25:09,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6b132942b031ffcc7f2f51c4c9e8428c in 1383ms, sequenceid=214, compaction requested=true 2024-11-21T03:25:09,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:09,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:09,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:09,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:09,612 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:09,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:09,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:09,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:25:09,612 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:09,615 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:09,615 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:09,615 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:09,615 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,615 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:09,615 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,615 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c555bfcfefdf4f87bec932644e979062, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/867bab1cad104d0da6580ec5270e7b64, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f52ecaa049fd409490565c5887e9d60e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/0cac5dc274724286a52a4286a6f929f4] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=50.1 K 2024-11-21T03:25:09,615 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7c83412a47354a15af78da939c195e5a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/4e8d5e74170942d0848606bfd05dff6f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/2afc9bfd7e5343a0aba161ab5b50a47e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/85d5635031e94c7387e073088de6495f] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=47.8 K 2024-11-21T03:25:09,616 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c555bfcfefdf4f87bec932644e979062, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732159506268 2024-11-21T03:25:09,616 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c83412a47354a15af78da939c195e5a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732159506268 2024-11-21T03:25:09,620 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e8d5e74170942d0848606bfd05dff6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732159506936 2024-11-21T03:25:09,621 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2afc9bfd7e5343a0aba161ab5b50a47e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732159506983 2024-11-21T03:25:09,621 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 85d5635031e94c7387e073088de6495f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159508163 2024-11-21T03:25:09,624 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 867bab1cad104d0da6580ec5270e7b64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732159506936 2024-11-21T03:25:09,625 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f52ecaa049fd409490565c5887e9d60e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732159506983 2024-11-21T03:25:09,625 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cac5dc274724286a52a4286a6f929f4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159508163 2024-11-21T03:25:09,643 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:09,643 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/092aac1a2b3a49e7bc6f5fcfee7ae3f3 is 50, key is test_row_0/B:col10/1732159508226/Put/seqid=0 2024-11-21T03:25:09,646 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:09,646 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/4420da4d37734de1801d2e4d48d4ccc9 is 50, key is test_row_0/A:col10/1732159508226/Put/seqid=0 2024-11-21T03:25:09,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742114_1290 (size=12629) 2024-11-21T03:25:09,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742113_1289 (size=12629) 2024-11-21T03:25:09,677 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/4420da4d37734de1801d2e4d48d4ccc9 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4420da4d37734de1801d2e4d48d4ccc9 2024-11-21T03:25:09,683 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/092aac1a2b3a49e7bc6f5fcfee7ae3f3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/092aac1a2b3a49e7bc6f5fcfee7ae3f3 2024-11-21T03:25:09,688 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into 4420da4d37734de1801d2e4d48d4ccc9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:09,688 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:09,688 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=12, startTime=1732159509611; duration=0sec 2024-11-21T03:25:09,688 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:09,688 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:09,689 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:09,693 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:09,693 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:09,693 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,693 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/25b6a047a3474f08b49dd76f4c54dea7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0f0a05571e549d298c7bd6b8035ed75, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/995c00fb7b1c42a0af737c6ca7aaa29d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65860daa1a9e4dfca52f127ccd18a0e8] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=47.8 K 2024-11-21T03:25:09,694 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25b6a047a3474f08b49dd76f4c54dea7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732159506268 2024-11-21T03:25:09,695 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f0a05571e549d298c7bd6b8035ed75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732159506936 2024-11-21T03:25:09,695 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into 092aac1a2b3a49e7bc6f5fcfee7ae3f3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:09,695 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:09,695 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=12, startTime=1732159509611; duration=0sec 2024-11-21T03:25:09,695 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:09,695 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:09,696 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 995c00fb7b1c42a0af737c6ca7aaa29d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732159506983 2024-11-21T03:25:09,696 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65860daa1a9e4dfca52f127ccd18a0e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159508163 2024-11-21T03:25:09,706 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:09,707 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/8723dac80c3441db8d9f10dd90a5ca08 is 50, key is test_row_0/C:col10/1732159508226/Put/seqid=0 2024-11-21T03:25:09,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T03:25:09,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,732 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:25:09,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/19e42dc476ff4e38949216d31ab55cf3 is 50, key is test_row_0/A:col10/1732159508286/Put/seqid=0 2024-11-21T03:25:09,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742115_1291 (size=12629) 2024-11-21T03:25:09,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742116_1292 (size=12151) 2024-11-21T03:25:09,777 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/19e42dc476ff4e38949216d31ab55cf3 2024-11-21T03:25:09,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/90083ccac6074dbfa6017ef8811825ce is 50, key is test_row_0/B:col10/1732159508286/Put/seqid=0 2024-11-21T03:25:09,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742117_1293 (size=12151) 2024-11-21T03:25:09,842 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/90083ccac6074dbfa6017ef8811825ce 2024-11-21T03:25:09,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/03c0f68f26414e7fa0fcf193014e45b3 is 50, key is test_row_0/C:col10/1732159508286/Put/seqid=0 2024-11-21T03:25:09,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742118_1294 (size=12151) 2024-11-21T03:25:09,906 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/03c0f68f26414e7fa0fcf193014e45b3 2024-11-21T03:25:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T03:25:09,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/19e42dc476ff4e38949216d31ab55cf3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/19e42dc476ff4e38949216d31ab55cf3 2024-11-21T03:25:09,949 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/19e42dc476ff4e38949216d31ab55cf3, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T03:25:09,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/90083ccac6074dbfa6017ef8811825ce as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/90083ccac6074dbfa6017ef8811825ce 2024-11-21T03:25:09,974 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/90083ccac6074dbfa6017ef8811825ce, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T03:25:09,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/03c0f68f26414e7fa0fcf193014e45b3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/03c0f68f26414e7fa0fcf193014e45b3 2024-11-21T03:25:09,984 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/03c0f68f26414e7fa0fcf193014e45b3, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T03:25:09,988 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 6b132942b031ffcc7f2f51c4c9e8428c in 256ms, sequenceid=240, compaction requested=false 2024-11-21T03:25:09,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:09,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:09,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-21T03:25:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-21T03:25:10,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-21T03:25:10,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2220 sec 2024-11-21T03:25:10,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.2390 sec 2024-11-21T03:25:10,163 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/8723dac80c3441db8d9f10dd90a5ca08 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/8723dac80c3441db8d9f10dd90a5ca08 2024-11-21T03:25:10,176 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 8723dac80c3441db8d9f10dd90a5ca08(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:10,176 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:10,176 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=12, startTime=1732159509612; duration=0sec 2024-11-21T03:25:10,176 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:10,176 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:10,477 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:10,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:25:10,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/8d9aef9f20aa4f61a0895e5f767e2256 is 50, key is test_row_0/A:col10/1732159510443/Put/seqid=0 2024-11-21T03:25:10,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742119_1295 (size=14541) 2024-11-21T03:25:10,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159570503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159570504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159570507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159570507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159570508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159570620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159570620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159570620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159570620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159570621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159570828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159570828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159570828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159570828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159570828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:10,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:10,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/8d9aef9f20aa4f61a0895e5f767e2256 2024-11-21T03:25:10,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/59f7523eb19947ad906818f5dbbd64f4 is 50, key is test_row_0/B:col10/1732159510443/Put/seqid=0 2024-11-21T03:25:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T03:25:10,917 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:10,920 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-21T03:25:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T03:25:10,922 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:10,923 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:10,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:10,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742120_1296 (size=12151) 2024-11-21T03:25:11,016 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T03:25:11,016 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T03:25:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T03:25:11,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T03:25:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159571138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:11,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159571138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:11,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159571138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:11,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159571138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:11,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159571138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:11,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:11,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T03:25:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T03:25:11,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:11,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/59f7523eb19947ad906818f5dbbd64f4 2024-11-21T03:25:11,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/ebeaf54616234b42abce6199e2bf496e is 50, key is test_row_0/C:col10/1732159510443/Put/seqid=0 2024-11-21T03:25:11,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742121_1297 (size=12151) 2024-11-21T03:25:11,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T03:25:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T03:25:11,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T03:25:11,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:11,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159571650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:11,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159571650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:11,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159571652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159571652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:11,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:11,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159571657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:11,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:11,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T03:25:11,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:11,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:11,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/ebeaf54616234b42abce6199e2bf496e 2024-11-21T03:25:11,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/8d9aef9f20aa4f61a0895e5f767e2256 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8d9aef9f20aa4f61a0895e5f767e2256 2024-11-21T03:25:11,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8d9aef9f20aa4f61a0895e5f767e2256, entries=200, sequenceid=253, filesize=14.2 K 2024-11-21T03:25:11,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/59f7523eb19947ad906818f5dbbd64f4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/59f7523eb19947ad906818f5dbbd64f4 2024-11-21T03:25:11,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/59f7523eb19947ad906818f5dbbd64f4, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T03:25:11,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/ebeaf54616234b42abce6199e2bf496e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ebeaf54616234b42abce6199e2bf496e 2024-11-21T03:25:11,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ebeaf54616234b42abce6199e2bf496e, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T03:25:11,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6b132942b031ffcc7f2f51c4c9e8428c in 1365ms, sequenceid=253, compaction requested=true 2024-11-21T03:25:11,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:11,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:11,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:11,842 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:11,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:11,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:11,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:11,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:25:11,843 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:11,844 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:11,844 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:11,844 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:11,844 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:11,844 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,844 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,844 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4420da4d37734de1801d2e4d48d4ccc9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/19e42dc476ff4e38949216d31ab55cf3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8d9aef9f20aa4f61a0895e5f767e2256] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=38.4 K 2024-11-21T03:25:11,844 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/092aac1a2b3a49e7bc6f5fcfee7ae3f3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/90083ccac6074dbfa6017ef8811825ce, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/59f7523eb19947ad906818f5dbbd64f4] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=36.1 K 2024-11-21T03:25:11,844 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4420da4d37734de1801d2e4d48d4ccc9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159508163 2024-11-21T03:25:11,844 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 092aac1a2b3a49e7bc6f5fcfee7ae3f3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159508163 2024-11-21T03:25:11,845 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19e42dc476ff4e38949216d31ab55cf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732159508282 2024-11-21T03:25:11,846 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 90083ccac6074dbfa6017ef8811825ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732159508282 2024-11-21T03:25:11,846 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d9aef9f20aa4f61a0895e5f767e2256, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732159510443 2024-11-21T03:25:11,846 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 59f7523eb19947ad906818f5dbbd64f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732159510443 2024-11-21T03:25:11,857 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:11,858 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/84cb98249a564d419e7490dbf54e4595 is 50, key is test_row_0/B:col10/1732159510443/Put/seqid=0 2024-11-21T03:25:11,858 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:11,859 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/d0127504cf3847b7aa2e26152059ea7d is 50, key is test_row_0/A:col10/1732159510443/Put/seqid=0 2024-11-21T03:25:11,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742122_1298 (size=12731) 2024-11-21T03:25:11,882 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/84cb98249a564d419e7490dbf54e4595 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/84cb98249a564d419e7490dbf54e4595 2024-11-21T03:25:11,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T03:25:11,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,885 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:25:11,889 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into 84cb98249a564d419e7490dbf54e4595(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:11,889 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:11,889 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=13, startTime=1732159511842; duration=0sec 2024-11-21T03:25:11,889 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:11,889 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:11,890 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:11,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/4d0184ba789e4f9ea1d30aaf75b360e5 is 50, key is test_row_0/A:col10/1732159510506/Put/seqid=0 2024-11-21T03:25:11,893 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:11,893 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:11,893 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:11,893 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/8723dac80c3441db8d9f10dd90a5ca08, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/03c0f68f26414e7fa0fcf193014e45b3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ebeaf54616234b42abce6199e2bf496e] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=36.1 K 2024-11-21T03:25:11,893 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8723dac80c3441db8d9f10dd90a5ca08, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732159508163 2024-11-21T03:25:11,894 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c0f68f26414e7fa0fcf193014e45b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732159508282 2024-11-21T03:25:11,894 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting ebeaf54616234b42abce6199e2bf496e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732159510443 2024-11-21T03:25:11,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742123_1299 (size=12731) 2024-11-21T03:25:11,903 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/d0127504cf3847b7aa2e26152059ea7d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d0127504cf3847b7aa2e26152059ea7d 2024-11-21T03:25:11,911 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into d0127504cf3847b7aa2e26152059ea7d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:11,912 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:11,912 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=13, startTime=1732159511842; duration=0sec 2024-11-21T03:25:11,912 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:11,912 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:11,921 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:11,922 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/0a041152f36b4c0fac0b67575e7251ff is 50, key is test_row_0/C:col10/1732159510443/Put/seqid=0 2024-11-21T03:25:11,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742124_1300 (size=12301) 2024-11-21T03:25:11,925 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/4d0184ba789e4f9ea1d30aaf75b360e5 2024-11-21T03:25:11,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742125_1301 (size=12731) 2024-11-21T03:25:11,946 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/0a041152f36b4c0fac0b67575e7251ff as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a041152f36b4c0fac0b67575e7251ff 2024-11-21T03:25:11,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/c2c97a405703441d86ef055147dbf794 is 50, key is test_row_0/B:col10/1732159510506/Put/seqid=0 2024-11-21T03:25:11,952 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 0a041152f36b4c0fac0b67575e7251ff(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:11,952 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:11,952 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=13, startTime=1732159511843; duration=0sec 2024-11-21T03:25:11,952 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:11,952 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:11,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742126_1302 (size=12301) 2024-11-21T03:25:11,966 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/c2c97a405703441d86ef055147dbf794 2024-11-21T03:25:11,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/9b2a7cfefaca4a80823ed4423c8b11ef is 50, key is test_row_0/C:col10/1732159510506/Put/seqid=0 2024-11-21T03:25:12,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742127_1303 (size=12301) 2024-11-21T03:25:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T03:25:12,417 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/9b2a7cfefaca4a80823ed4423c8b11ef 2024-11-21T03:25:12,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/4d0184ba789e4f9ea1d30aaf75b360e5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4d0184ba789e4f9ea1d30aaf75b360e5 2024-11-21T03:25:12,426 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4d0184ba789e4f9ea1d30aaf75b360e5, entries=150, sequenceid=277, filesize=12.0 K 2024-11-21T03:25:12,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/c2c97a405703441d86ef055147dbf794 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c2c97a405703441d86ef055147dbf794 2024-11-21T03:25:12,431 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c2c97a405703441d86ef055147dbf794, entries=150, sequenceid=277, filesize=12.0 K 2024-11-21T03:25:12,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/9b2a7cfefaca4a80823ed4423c8b11ef as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9b2a7cfefaca4a80823ed4423c8b11ef 2024-11-21T03:25:12,439 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9b2a7cfefaca4a80823ed4423c8b11ef, entries=150, sequenceid=277, filesize=12.0 K 2024-11-21T03:25:12,440 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 6b132942b031ffcc7f2f51c4c9e8428c in 554ms, sequenceid=277, compaction requested=false 2024-11-21T03:25:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-21T03:25:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-21T03:25:12,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-21T03:25:12,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5180 sec 2024-11-21T03:25:12,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5240 sec 2024-11-21T03:25:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:12,675 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:12,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:25:12,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/bd09f66802074f809c9e777c45e61a5a is 50, key is test_row_1/A:col10/1732159512674/Put/seqid=0 2024-11-21T03:25:12,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742128_1304 (size=14737) 2024-11-21T03:25:12,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159572711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159572711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:12,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159572711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:12,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159572712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:12,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159572713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:12,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159572819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:12,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159572819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:12,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159572819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:12,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159572819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:12,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:12,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:12,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159572828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159573030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159573030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159573030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159573038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159573038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T03:25:13,089 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:13,091 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:13,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-21T03:25:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T03:25:13,096 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:13,100 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:13,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:13,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/bd09f66802074f809c9e777c45e61a5a 2024-11-21T03:25:13,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/370dd6fbebef469295026b5ac09bcbae is 50, key is test_row_1/B:col10/1732159512674/Put/seqid=0 2024-11-21T03:25:13,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742129_1305 (size=9857) 2024-11-21T03:25:13,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/370dd6fbebef469295026b5ac09bcbae 2024-11-21T03:25:13,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/5533a71e0046409abbbf211e946c0c5b is 50, key is test_row_1/C:col10/1732159512674/Put/seqid=0 2024-11-21T03:25:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T03:25:13,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742130_1306 (size=9857) 2024-11-21T03:25:13,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/5533a71e0046409abbbf211e946c0c5b 2024-11-21T03:25:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/bd09f66802074f809c9e777c45e61a5a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/bd09f66802074f809c9e777c45e61a5a 2024-11-21T03:25:13,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/bd09f66802074f809c9e777c45e61a5a, entries=200, sequenceid=291, filesize=14.4 K 2024-11-21T03:25:13,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/370dd6fbebef469295026b5ac09bcbae as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/370dd6fbebef469295026b5ac09bcbae 2024-11-21T03:25:13,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/370dd6fbebef469295026b5ac09bcbae, entries=100, sequenceid=291, filesize=9.6 K 2024-11-21T03:25:13,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/5533a71e0046409abbbf211e946c0c5b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/5533a71e0046409abbbf211e946c0c5b 2024-11-21T03:25:13,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/5533a71e0046409abbbf211e946c0c5b, entries=100, sequenceid=291, filesize=9.6 K 2024-11-21T03:25:13,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6b132942b031ffcc7f2f51c4c9e8428c in 664ms, sequenceid=291, compaction requested=true 2024-11-21T03:25:13,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:13,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:13,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:13,340 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:13,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:13,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:13,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:13,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:25:13,342 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:13,347 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:13,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T03:25:13,350 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:13,351 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:13,351 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,351 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/84cb98249a564d419e7490dbf54e4595, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c2c97a405703441d86ef055147dbf794, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/370dd6fbebef469295026b5ac09bcbae] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=34.1 K 2024-11-21T03:25:13,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,356 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 84cb98249a564d419e7490dbf54e4595, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732159510443 2024-11-21T03:25:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159573350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,359 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c97a405703441d86ef055147dbf794, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732159510493 2024-11-21T03:25:13,360 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39769 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:13,360 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:13,360 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,360 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d0127504cf3847b7aa2e26152059ea7d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4d0184ba789e4f9ea1d30aaf75b360e5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/bd09f66802074f809c9e777c45e61a5a] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=38.8 K 2024-11-21T03:25:13,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159573354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159573354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159573355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159573356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,368 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 370dd6fbebef469295026b5ac09bcbae, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732159512672 2024-11-21T03:25:13,371 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0127504cf3847b7aa2e26152059ea7d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732159510443 2024-11-21T03:25:13,374 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d0184ba789e4f9ea1d30aaf75b360e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732159510493 2024-11-21T03:25:13,377 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd09f66802074f809c9e777c45e61a5a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732159512669 2024-11-21T03:25:13,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/df767503554340049ce5c54fe958f592 is 50, key is test_row_0/A:col10/1732159513346/Put/seqid=0 2024-11-21T03:25:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T03:25:13,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:13,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:13,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,448 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:13,449 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/537bf96093984986a2a0dedc4c2857c2 is 50, key is test_row_0/B:col10/1732159510506/Put/seqid=0 2024-11-21T03:25:13,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742131_1307 (size=12301) 2024-11-21T03:25:13,464 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:13,465 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/5336aca212a04c13b589dec6c636e7aa is 50, key is test_row_0/A:col10/1732159510506/Put/seqid=0 2024-11-21T03:25:13,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159573469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159573469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159573469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159573469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159573470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742132_1308 (size=12983) 2024-11-21T03:25:13,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742133_1309 (size=12983) 2024-11-21T03:25:13,552 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/5336aca212a04c13b589dec6c636e7aa as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5336aca212a04c13b589dec6c636e7aa 2024-11-21T03:25:13,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,609 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into 5336aca212a04c13b589dec6c636e7aa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:13,609 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:13,609 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=13, startTime=1732159513340; duration=0sec 2024-11-21T03:25:13,609 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:13,609 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:13,610 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:13,620 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:13,620 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:13,621 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,621 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a041152f36b4c0fac0b67575e7251ff, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9b2a7cfefaca4a80823ed4423c8b11ef, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/5533a71e0046409abbbf211e946c0c5b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=34.1 K 2024-11-21T03:25:13,624 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a041152f36b4c0fac0b67575e7251ff, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732159510443 2024-11-21T03:25:13,628 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b2a7cfefaca4a80823ed4423c8b11ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732159510493 2024-11-21T03:25:13,632 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5533a71e0046409abbbf211e946c0c5b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732159512672 2024-11-21T03:25:13,679 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:13,679 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/9e178e1302b945cfadd971089a2756a0 is 50, key is test_row_0/C:col10/1732159510506/Put/seqid=0 2024-11-21T03:25:13,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159573680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159573681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159573681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159573681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159573685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T03:25:13,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742134_1310 (size=12983) 2024-11-21T03:25:13,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:13,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:13,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/df767503554340049ce5c54fe958f592 2024-11-21T03:25:13,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/7e28559a71044078b7b4794c9f745070 is 50, key is test_row_0/B:col10/1732159513346/Put/seqid=0 2024-11-21T03:25:13,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742135_1311 (size=12301) 2024-11-21T03:25:13,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/7e28559a71044078b7b4794c9f745070 2024-11-21T03:25:13,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d62245deda2848abb03ea2ac76c8808d is 50, key is test_row_0/C:col10/1732159513346/Put/seqid=0 2024-11-21T03:25:13,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742136_1312 (size=12301) 2024-11-21T03:25:13,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d62245deda2848abb03ea2ac76c8808d 2024-11-21T03:25:13,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:13,918 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/537bf96093984986a2a0dedc4c2857c2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/537bf96093984986a2a0dedc4c2857c2 2024-11-21T03:25:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:13,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/df767503554340049ce5c54fe958f592 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/df767503554340049ce5c54fe958f592 2024-11-21T03:25:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:13,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/df767503554340049ce5c54fe958f592, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T03:25:13,933 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into 537bf96093984986a2a0dedc4c2857c2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:13,933 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:13,933 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=13, startTime=1732159513340; duration=0sec 2024-11-21T03:25:13,934 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:13,934 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:13,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/7e28559a71044078b7b4794c9f745070 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7e28559a71044078b7b4794c9f745070 2024-11-21T03:25:13,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7e28559a71044078b7b4794c9f745070, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T03:25:13,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d62245deda2848abb03ea2ac76c8808d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d62245deda2848abb03ea2ac76c8808d 2024-11-21T03:25:13,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d62245deda2848abb03ea2ac76c8808d, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T03:25:13,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 6b132942b031ffcc7f2f51c4c9e8428c in 637ms, sequenceid=320, compaction requested=false 2024-11-21T03:25:13,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:14,003 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:14,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:25:14,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/57f0cdd900c6411589cf7062eac8bae7 is 50, key is test_row_0/A:col10/1732159514001/Put/seqid=0 2024-11-21T03:25:14,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742137_1313 (size=17181) 2024-11-21T03:25:14,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/57f0cdd900c6411589cf7062eac8bae7 2024-11-21T03:25:14,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/1a98d1dc9c8d41d08c09f8d27db0f183 is 50, key is test_row_0/B:col10/1732159514001/Put/seqid=0 2024-11-21T03:25:14,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159574037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159574042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159574044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159574046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159574046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742138_1314 (size=12301) 2024-11-21T03:25:14,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:14,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:14,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,138 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/9e178e1302b945cfadd971089a2756a0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9e178e1302b945cfadd971089a2756a0 2024-11-21T03:25:14,143 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 9e178e1302b945cfadd971089a2756a0(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:14,143 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:14,143 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=13, startTime=1732159513340; duration=0sec 2024-11-21T03:25:14,143 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:14,143 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159574158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159574159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159574160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159574160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159574160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:14,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:14,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T03:25:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159574368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159574368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159574368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159574369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159574377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/1a98d1dc9c8d41d08c09f8d27db0f183 2024-11-21T03:25:14,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/eb2755e1df3747ac8596770b0484c32e is 50, key is test_row_0/C:col10/1732159514001/Put/seqid=0 2024-11-21T03:25:14,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742139_1315 (size=12301) 2024-11-21T03:25:14,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/eb2755e1df3747ac8596770b0484c32e 2024-11-21T03:25:14,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/57f0cdd900c6411589cf7062eac8bae7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/57f0cdd900c6411589cf7062eac8bae7 2024-11-21T03:25:14,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/57f0cdd900c6411589cf7062eac8bae7, entries=250, sequenceid=334, filesize=16.8 K 2024-11-21T03:25:14,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/1a98d1dc9c8d41d08c09f8d27db0f183 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a98d1dc9c8d41d08c09f8d27db0f183 2024-11-21T03:25:14,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a98d1dc9c8d41d08c09f8d27db0f183, entries=150, sequenceid=334, filesize=12.0 K 2024-11-21T03:25:14,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/eb2755e1df3747ac8596770b0484c32e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/eb2755e1df3747ac8596770b0484c32e 2024-11-21T03:25:14,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/eb2755e1df3747ac8596770b0484c32e, entries=150, sequenceid=334, filesize=12.0 K 2024-11-21T03:25:14,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6b132942b031ffcc7f2f51c4c9e8428c in 569ms, sequenceid=334, compaction requested=true 2024-11-21T03:25:14,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:14,573 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:14,574 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:14,574 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:14,574 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,575 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5336aca212a04c13b589dec6c636e7aa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/df767503554340049ce5c54fe958f592, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/57f0cdd900c6411589cf7062eac8bae7] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=41.5 K 2024-11-21T03:25:14,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:14,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:14,575 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:14,575 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5336aca212a04c13b589dec6c636e7aa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732159510506 2024-11-21T03:25:14,576 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting df767503554340049ce5c54fe958f592, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732159513345 2024-11-21T03:25:14,577 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57f0cdd900c6411589cf7062eac8bae7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732159513352 2024-11-21T03:25:14,577 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:14,578 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:14,578 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,578 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/537bf96093984986a2a0dedc4c2857c2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7e28559a71044078b7b4794c9f745070, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a98d1dc9c8d41d08c09f8d27db0f183] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=36.7 K 2024-11-21T03:25:14,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:14,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:14,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:14,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:14,579 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 537bf96093984986a2a0dedc4c2857c2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732159510506 2024-11-21T03:25:14,580 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e28559a71044078b7b4794c9f745070, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732159513345 2024-11-21T03:25:14,580 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a98d1dc9c8d41d08c09f8d27db0f183, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732159513999 2024-11-21T03:25:14,599 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:14,599 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/125c8e2b90ba4bb0ad10b8d8afdd11af is 50, key is test_row_0/A:col10/1732159514001/Put/seqid=0 2024-11-21T03:25:14,602 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:14,603 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/993def98ee6842daac72fc7f6cad49b6 is 50, key is test_row_0/B:col10/1732159514001/Put/seqid=0 2024-11-21T03:25:14,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742140_1316 (size=13085) 2024-11-21T03:25:14,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742141_1317 (size=13085) 2024-11-21T03:25:14,625 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/125c8e2b90ba4bb0ad10b8d8afdd11af as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/125c8e2b90ba4bb0ad10b8d8afdd11af 2024-11-21T03:25:14,633 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into 125c8e2b90ba4bb0ad10b8d8afdd11af(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:14,633 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:14,633 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=13, startTime=1732159514573; duration=0sec 2024-11-21T03:25:14,633 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:14,633 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:14,633 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:14,634 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:14,634 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:14,635 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,635 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9e178e1302b945cfadd971089a2756a0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d62245deda2848abb03ea2ac76c8808d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/eb2755e1df3747ac8596770b0484c32e] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=36.7 K 2024-11-21T03:25:14,635 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e178e1302b945cfadd971089a2756a0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732159510506 2024-11-21T03:25:14,636 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d62245deda2848abb03ea2ac76c8808d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732159513345 2024-11-21T03:25:14,637 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb2755e1df3747ac8596770b0484c32e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732159513999 2024-11-21T03:25:14,650 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:14,651 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/0c90b150d2ec419380192780ee71ae67 is 50, key is test_row_0/C:col10/1732159514001/Put/seqid=0 2024-11-21T03:25:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:14,680 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:14,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:25:14,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742142_1318 (size=13085) 2024-11-21T03:25:14,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/ffc5bf2b5fa340298165e7266efa21ae is 50, key is test_row_0/A:col10/1732159514679/Put/seqid=0 2024-11-21T03:25:14,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:14,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159574689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159574695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742143_1319 (size=12301) 2024-11-21T03:25:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159574695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/ffc5bf2b5fa340298165e7266efa21ae 2024-11-21T03:25:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,709 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/0c90b150d2ec419380192780ee71ae67 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0c90b150d2ec419380192780ee71ae67 2024-11-21T03:25:14,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159574702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159574702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/ce4f69cef4cd4350802c312b6a34be18 is 50, key is test_row_0/B:col10/1732159514679/Put/seqid=0 2024-11-21T03:25:14,743 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 0c90b150d2ec419380192780ee71ae67(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:14,743 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:14,743 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=13, startTime=1732159514579; duration=0sec 2024-11-21T03:25:14,743 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:14,743 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:14,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742144_1320 (size=12301) 2024-11-21T03:25:14,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/ce4f69cef4cd4350802c312b6a34be18 2024-11-21T03:25:14,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159574813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159574813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/7b8cb2f515634405848aa419d3a0936c is 50, key is test_row_0/C:col10/1732159514679/Put/seqid=0 2024-11-21T03:25:14,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159574822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159574822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159574823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:14,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:14,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:14,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:14,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742145_1321 (size=12301) 2024-11-21T03:25:15,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:15,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:15,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159575027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159575028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159575038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159575038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,044 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/993def98ee6842daac72fc7f6cad49b6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/993def98ee6842daac72fc7f6cad49b6 2024-11-21T03:25:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159575045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,056 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into 993def98ee6842daac72fc7f6cad49b6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:15,056 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:15,056 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=13, startTime=1732159514575; duration=0sec 2024-11-21T03:25:15,056 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:15,056 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:15,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:15,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T03:25:15,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/7b8cb2f515634405848aa419d3a0936c 2024-11-21T03:25:15,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/ffc5bf2b5fa340298165e7266efa21ae as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/ffc5bf2b5fa340298165e7266efa21ae 2024-11-21T03:25:15,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/ffc5bf2b5fa340298165e7266efa21ae, entries=150, sequenceid=361, filesize=12.0 K 2024-11-21T03:25:15,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/ce4f69cef4cd4350802c312b6a34be18 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/ce4f69cef4cd4350802c312b6a34be18 2024-11-21T03:25:15,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/ce4f69cef4cd4350802c312b6a34be18, entries=150, sequenceid=361, filesize=12.0 K 2024-11-21T03:25:15,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/7b8cb2f515634405848aa419d3a0936c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/7b8cb2f515634405848aa419d3a0936c 2024-11-21T03:25:15,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/7b8cb2f515634405848aa419d3a0936c, entries=150, sequenceid=361, filesize=12.0 K 2024-11-21T03:25:15,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6b132942b031ffcc7f2f51c4c9e8428c in 631ms, sequenceid=361, compaction requested=false 2024-11-21T03:25:15,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:15,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T03:25:15,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:15,327 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:25:15,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/e58f22b61e0b4e978bf3b5a7697ab26c is 50, key is test_row_0/A:col10/1732159514694/Put/seqid=0 2024-11-21T03:25:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:15,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:15,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742146_1322 (size=12301) 2024-11-21T03:25:15,366 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/e58f22b61e0b4e978bf3b5a7697ab26c 2024-11-21T03:25:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/b1c852aaf15342809a805a2d661f576f is 50, key is test_row_0/B:col10/1732159514694/Put/seqid=0 2024-11-21T03:25:15,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742147_1323 (size=12301) 2024-11-21T03:25:15,389 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/b1c852aaf15342809a805a2d661f576f 2024-11-21T03:25:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/0e63b88b00d743be8df9f836a9836f18 is 50, key is test_row_0/C:col10/1732159514694/Put/seqid=0 2024-11-21T03:25:15,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159575388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159575395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742148_1324 (size=12301) 2024-11-21T03:25:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159575399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159575402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159575402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,429 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/0e63b88b00d743be8df9f836a9836f18 2024-11-21T03:25:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/e58f22b61e0b4e978bf3b5a7697ab26c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e58f22b61e0b4e978bf3b5a7697ab26c 2024-11-21T03:25:15,462 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e58f22b61e0b4e978bf3b5a7697ab26c, entries=150, sequenceid=374, filesize=12.0 K 2024-11-21T03:25:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/b1c852aaf15342809a805a2d661f576f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b1c852aaf15342809a805a2d661f576f 2024-11-21T03:25:15,471 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b1c852aaf15342809a805a2d661f576f, entries=150, sequenceid=374, filesize=12.0 K 2024-11-21T03:25:15,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/0e63b88b00d743be8df9f836a9836f18 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0e63b88b00d743be8df9f836a9836f18 2024-11-21T03:25:15,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0e63b88b00d743be8df9f836a9836f18, entries=150, sequenceid=374, filesize=12.0 K 2024-11-21T03:25:15,490 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6b132942b031ffcc7f2f51c4c9e8428c in 163ms, sequenceid=374, compaction requested=true 2024-11-21T03:25:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-21T03:25:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-21T03:25:15,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-21T03:25:15,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3920 sec 2024-11-21T03:25:15,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.4040 sec 2024-11-21T03:25:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:15,554 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:15,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-21T03:25:15,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159575565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/12f098386bdc4bcc924f984aef6b99eb is 50, key is test_row_0/A:col10/1732159515397/Put/seqid=0 2024-11-21T03:25:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742149_1325 (size=12301) 2024-11-21T03:25:15,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159575568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/12f098386bdc4bcc924f984aef6b99eb 2024-11-21T03:25:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159575574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159575580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159575589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/6cc64f90558b423d81e0585bb6d3a79d is 50, key is test_row_0/B:col10/1732159515397/Put/seqid=0 2024-11-21T03:25:15,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742150_1326 (size=12301) 2024-11-21T03:25:15,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/6cc64f90558b423d81e0585bb6d3a79d 2024-11-21T03:25:15,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/2b6490b8af1c479f8d4a08b2e5625d16 is 50, key is test_row_0/C:col10/1732159515397/Put/seqid=0 2024-11-21T03:25:15,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742151_1327 (size=12301) 2024-11-21T03:25:15,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/2b6490b8af1c479f8d4a08b2e5625d16 2024-11-21T03:25:15,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159575688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159575697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159575698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159575698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159575698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:15,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/12f098386bdc4bcc924f984aef6b99eb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/12f098386bdc4bcc924f984aef6b99eb 2024-11-21T03:25:15,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/12f098386bdc4bcc924f984aef6b99eb, entries=150, sequenceid=403, filesize=12.0 K 2024-11-21T03:25:15,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/6cc64f90558b423d81e0585bb6d3a79d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/6cc64f90558b423d81e0585bb6d3a79d 2024-11-21T03:25:15,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/6cc64f90558b423d81e0585bb6d3a79d, entries=150, sequenceid=403, filesize=12.0 K 2024-11-21T03:25:15,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/2b6490b8af1c479f8d4a08b2e5625d16 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b6490b8af1c479f8d4a08b2e5625d16 2024-11-21T03:25:15,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b6490b8af1c479f8d4a08b2e5625d16, entries=150, sequenceid=403, filesize=12.0 K 2024-11-21T03:25:15,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 6b132942b031ffcc7f2f51c4c9e8428c in 197ms, sequenceid=403, compaction requested=true 2024-11-21T03:25:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:15,751 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:15,751 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:15,753 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:15,753 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:15,753 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:15,753 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:15,753 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:15,753 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:15,753 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/125c8e2b90ba4bb0ad10b8d8afdd11af, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/ffc5bf2b5fa340298165e7266efa21ae, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e58f22b61e0b4e978bf3b5a7697ab26c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/12f098386bdc4bcc924f984aef6b99eb] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=48.8 K 2024-11-21T03:25:15,753 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/993def98ee6842daac72fc7f6cad49b6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/ce4f69cef4cd4350802c312b6a34be18, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b1c852aaf15342809a805a2d661f576f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/6cc64f90558b423d81e0585bb6d3a79d] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=48.8 K 2024-11-21T03:25:15,753 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 993def98ee6842daac72fc7f6cad49b6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732159513999 2024-11-21T03:25:15,754 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 125c8e2b90ba4bb0ad10b8d8afdd11af, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732159513999 2024-11-21T03:25:15,754 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting ce4f69cef4cd4350802c312b6a34be18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732159514027 2024-11-21T03:25:15,754 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc5bf2b5fa340298165e7266efa21ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732159514027 2024-11-21T03:25:15,755 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c852aaf15342809a805a2d661f576f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732159514693 2024-11-21T03:25:15,755 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e58f22b61e0b4e978bf3b5a7697ab26c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732159514693 2024-11-21T03:25:15,755 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cc64f90558b423d81e0585bb6d3a79d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732159515397 2024-11-21T03:25:15,755 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12f098386bdc4bcc924f984aef6b99eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732159515397 2024-11-21T03:25:15,792 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:15,793 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/46c4f2f43eab40af90d519c3d4fa7fd3 is 50, key is test_row_0/B:col10/1732159515397/Put/seqid=0 2024-11-21T03:25:15,803 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:15,808 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/daddb7f4b13849ed999b44ea06f3d98e is 50, key is test_row_0/A:col10/1732159515397/Put/seqid=0 2024-11-21T03:25:15,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742152_1328 (size=13221) 2024-11-21T03:25:15,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742153_1329 (size=13221) 2024-11-21T03:25:15,854 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/daddb7f4b13849ed999b44ea06f3d98e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/daddb7f4b13849ed999b44ea06f3d98e 2024-11-21T03:25:15,873 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into daddb7f4b13849ed999b44ea06f3d98e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:15,873 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:15,873 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=12, startTime=1732159515751; duration=0sec 2024-11-21T03:25:15,874 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:15,874 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:15,874 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:15,876 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:15,876 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:15,876 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:15,876 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0c90b150d2ec419380192780ee71ae67, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/7b8cb2f515634405848aa419d3a0936c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0e63b88b00d743be8df9f836a9836f18, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b6490b8af1c479f8d4a08b2e5625d16] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=48.8 K 2024-11-21T03:25:15,877 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c90b150d2ec419380192780ee71ae67, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732159513999 2024-11-21T03:25:15,877 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b8cb2f515634405848aa419d3a0936c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732159514027 2024-11-21T03:25:15,878 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e63b88b00d743be8df9f836a9836f18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732159514693 2024-11-21T03:25:15,878 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b6490b8af1c479f8d4a08b2e5625d16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732159515397 2024-11-21T03:25:15,917 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#281 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:15,918 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/908c832d0693484db32695b2cde74729 is 50, key is test_row_0/C:col10/1732159515397/Put/seqid=0 2024-11-21T03:25:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:15,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742154_1330 (size=13221) 2024-11-21T03:25:15,931 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:15,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:25:15,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/5d7386e2863e48ee8a660a7831027a60 is 50, key is test_row_0/A:col10/1732159515913/Put/seqid=0 2024-11-21T03:25:15,944 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/908c832d0693484db32695b2cde74729 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/908c832d0693484db32695b2cde74729 2024-11-21T03:25:15,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742155_1331 (size=14741) 2024-11-21T03:25:15,983 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 908c832d0693484db32695b2cde74729(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:15,983 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:15,983 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=12, startTime=1732159515751; duration=0sec 2024-11-21T03:25:15,983 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:15,983 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:16,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159576033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159576035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159576036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159576042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159576042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159576148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159576148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159576148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159576158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159576158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,241 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/46c4f2f43eab40af90d519c3d4fa7fd3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/46c4f2f43eab40af90d519c3d4fa7fd3 2024-11-21T03:25:16,260 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into 46c4f2f43eab40af90d519c3d4fa7fd3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:16,260 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:16,260 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=12, startTime=1732159515751; duration=0sec 2024-11-21T03:25:16,260 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:16,260 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:16,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/5d7386e2863e48ee8a660a7831027a60 2024-11-21T03:25:16,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159576360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159576360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159576368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159576368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159576377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/800b44dbda184d02b89d71719d399d98 is 50, key is test_row_0/B:col10/1732159515913/Put/seqid=0 2024-11-21T03:25:16,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742156_1332 (size=12301) 2024-11-21T03:25:16,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159576668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159576678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159576680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159576687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159576687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:16,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:16,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/800b44dbda184d02b89d71719d399d98 2024-11-21T03:25:16,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d5a45024ae8f4332a9e901ac436cd7be is 50, key is test_row_0/C:col10/1732159515913/Put/seqid=0 2024-11-21T03:25:16,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742157_1333 (size=12301) 2024-11-21T03:25:16,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d5a45024ae8f4332a9e901ac436cd7be 2024-11-21T03:25:16,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/5d7386e2863e48ee8a660a7831027a60 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5d7386e2863e48ee8a660a7831027a60 2024-11-21T03:25:16,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5d7386e2863e48ee8a660a7831027a60, entries=200, sequenceid=417, filesize=14.4 K 2024-11-21T03:25:16,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/800b44dbda184d02b89d71719d399d98 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/800b44dbda184d02b89d71719d399d98 2024-11-21T03:25:16,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/800b44dbda184d02b89d71719d399d98, entries=150, sequenceid=417, filesize=12.0 K 2024-11-21T03:25:16,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d5a45024ae8f4332a9e901ac436cd7be as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d5a45024ae8f4332a9e901ac436cd7be 2024-11-21T03:25:16,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d5a45024ae8f4332a9e901ac436cd7be, entries=150, sequenceid=417, filesize=12.0 K 2024-11-21T03:25:16,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 6b132942b031ffcc7f2f51c4c9e8428c in 1009ms, sequenceid=417, compaction requested=false 2024-11-21T03:25:16,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:17,188 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:17,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:25:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:17,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/e097524deefa444fa0bd7ab858e8367c is 50, key is test_row_0/A:col10/1732159516037/Put/seqid=0 2024-11-21T03:25:17,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159577227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159577227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159577228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159577228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159577229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742158_1334 (size=14741) 2024-11-21T03:25:17,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/e097524deefa444fa0bd7ab858e8367c 2024-11-21T03:25:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T03:25:17,258 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:17,259 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-21T03:25:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:25:17,268 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:17,269 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:17,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:17,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/1a8cbe4f21d2439689a5f7bbac041cb3 is 50, key is test_row_0/B:col10/1732159516037/Put/seqid=0 2024-11-21T03:25:17,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742159_1335 (size=12301) 2024-11-21T03:25:17,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/1a8cbe4f21d2439689a5f7bbac041cb3 2024-11-21T03:25:17,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/cc98ccaf17ea421a854179d44b777b67 is 50, key is test_row_0/C:col10/1732159516037/Put/seqid=0 2024-11-21T03:25:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159577338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159577338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159577338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159577338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159577338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:25:17,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742160_1336 (size=12301) 2024-11-21T03:25:17,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:25:17,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:17,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:17,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:17,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159577548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159577548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159577548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159577548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159577548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:25:17,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:17,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:17,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:17,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:25:17,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:25:17,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:17,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:17,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/cc98ccaf17ea421a854179d44b777b67 2024-11-21T03:25:17,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/e097524deefa444fa0bd7ab858e8367c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e097524deefa444fa0bd7ab858e8367c 2024-11-21T03:25:17,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e097524deefa444fa0bd7ab858e8367c, entries=200, sequenceid=444, filesize=14.4 K 2024-11-21T03:25:17,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/1a8cbe4f21d2439689a5f7bbac041cb3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a8cbe4f21d2439689a5f7bbac041cb3 2024-11-21T03:25:17,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a8cbe4f21d2439689a5f7bbac041cb3, entries=150, sequenceid=444, filesize=12.0 K 2024-11-21T03:25:17,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/cc98ccaf17ea421a854179d44b777b67 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/cc98ccaf17ea421a854179d44b777b67 2024-11-21T03:25:17,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/cc98ccaf17ea421a854179d44b777b67, entries=150, sequenceid=444, filesize=12.0 K 2024-11-21T03:25:17,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 6b132942b031ffcc7f2f51c4c9e8428c in 655ms, sequenceid=444, compaction requested=true 2024-11-21T03:25:17,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:17,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:17,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:17,844 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:17,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:17,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:17,844 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:17,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:17,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:17,845 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:17,845 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:17,845 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:17,845 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/46c4f2f43eab40af90d519c3d4fa7fd3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/800b44dbda184d02b89d71719d399d98, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a8cbe4f21d2439689a5f7bbac041cb3] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=36.9 K 2024-11-21T03:25:17,846 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:17,846 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:17,846 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:17,846 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/daddb7f4b13849ed999b44ea06f3d98e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5d7386e2863e48ee8a660a7831027a60, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e097524deefa444fa0bd7ab858e8367c] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=41.7 K 2024-11-21T03:25:17,846 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting daddb7f4b13849ed999b44ea06f3d98e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732159515397 2024-11-21T03:25:17,846 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 46c4f2f43eab40af90d519c3d4fa7fd3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732159515397 2024-11-21T03:25:17,847 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d7386e2863e48ee8a660a7831027a60, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732159515910 2024-11-21T03:25:17,847 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 800b44dbda184d02b89d71719d399d98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732159515913 2024-11-21T03:25:17,847 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e097524deefa444fa0bd7ab858e8367c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732159516033 2024-11-21T03:25:17,847 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a8cbe4f21d2439689a5f7bbac041cb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732159516033 2024-11-21T03:25:17,857 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:17,857 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/cd4d28738e0141fe8e000153dbf24a57 is 50, key is test_row_0/A:col10/1732159516037/Put/seqid=0 2024-11-21T03:25:17,861 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:17,861 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/18d95da6e3444cfdb13eae88a9959c8c is 50, key is test_row_0/B:col10/1732159516037/Put/seqid=0 2024-11-21T03:25:17,861 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:17,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:25:17,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742161_1337 (size=13323) 2024-11-21T03:25:17,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742162_1338 (size=13323) 2024-11-21T03:25:17,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/833eca263c124929b72dca14ff057c9b is 50, key is test_row_0/A:col10/1732159517861/Put/seqid=0 2024-11-21T03:25:17,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742163_1339 (size=12301) 2024-11-21T03:25:17,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/833eca263c124929b72dca14ff057c9b 2024-11-21T03:25:17,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:25:17,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:17,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:17,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:17,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:17,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/72f19f2f1d8b4013b05f4a83b56e57f4 is 50, key is test_row_0/B:col10/1732159517861/Put/seqid=0 2024-11-21T03:25:17,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742164_1340 (size=12301) 2024-11-21T03:25:17,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159577905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159577908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159577908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:25:17,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159577914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:17,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159577914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:17,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:17,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:17,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159578018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159578018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159578020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159578028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159578029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:25:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:25:18,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:18,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159578228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159578228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159578240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159578240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159578240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/72f19f2f1d8b4013b05f4a83b56e57f4 2024-11-21T03:25:18,305 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/cd4d28738e0141fe8e000153dbf24a57 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/cd4d28738e0141fe8e000153dbf24a57 2024-11-21T03:25:18,308 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/18d95da6e3444cfdb13eae88a9959c8c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/18d95da6e3444cfdb13eae88a9959c8c 2024-11-21T03:25:18,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d575e584f2e4490589d0e649a8276e52 is 50, key is test_row_0/C:col10/1732159517861/Put/seqid=0 2024-11-21T03:25:18,323 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into cd4d28738e0141fe8e000153dbf24a57(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:18,323 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:18,323 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=13, startTime=1732159517843; duration=0sec 2024-11-21T03:25:18,323 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:18,324 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:18,324 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:18,327 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into 18d95da6e3444cfdb13eae88a9959c8c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:18,327 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:18,327 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=13, startTime=1732159517844; duration=0sec 2024-11-21T03:25:18,327 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:18,327 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:18,330 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:18,330 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:18,330 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,330 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/908c832d0693484db32695b2cde74729, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d5a45024ae8f4332a9e901ac436cd7be, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/cc98ccaf17ea421a854179d44b777b67] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=36.9 K 2024-11-21T03:25:18,331 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 908c832d0693484db32695b2cde74729, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732159515397 2024-11-21T03:25:18,331 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a45024ae8f4332a9e901ac436cd7be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732159515913 2024-11-21T03:25:18,331 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc98ccaf17ea421a854179d44b777b67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732159516033 2024-11-21T03:25:18,346 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:18,347 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/59beceb33d114356ade87de63551aee4 is 50, key is test_row_0/C:col10/1732159516037/Put/seqid=0 2024-11-21T03:25:18,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742165_1341 (size=12301) 2024-11-21T03:25:18,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:25:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742166_1342 (size=13323) 2024-11-21T03:25:18,381 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/59beceb33d114356ade87de63551aee4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/59beceb33d114356ade87de63551aee4 2024-11-21T03:25:18,387 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 59beceb33d114356ade87de63551aee4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:18,387 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:18,387 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=13, startTime=1732159517844; duration=0sec 2024-11-21T03:25:18,387 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:18,388 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:25:18,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:25:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159578538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159578538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159578560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159578560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159578568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:18,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:25:18,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:18,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:18,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d575e584f2e4490589d0e649a8276e52 2024-11-21T03:25:18,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/833eca263c124929b72dca14ff057c9b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/833eca263c124929b72dca14ff057c9b 2024-11-21T03:25:18,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/833eca263c124929b72dca14ff057c9b, entries=150, sequenceid=460, filesize=12.0 K 2024-11-21T03:25:18,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/72f19f2f1d8b4013b05f4a83b56e57f4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/72f19f2f1d8b4013b05f4a83b56e57f4 2024-11-21T03:25:18,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/72f19f2f1d8b4013b05f4a83b56e57f4, entries=150, sequenceid=460, filesize=12.0 K 2024-11-21T03:25:18,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d575e584f2e4490589d0e649a8276e52 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d575e584f2e4490589d0e649a8276e52 2024-11-21T03:25:18,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d575e584f2e4490589d0e649a8276e52, entries=150, sequenceid=460, filesize=12.0 K 2024-11-21T03:25:18,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 6b132942b031ffcc7f2f51c4c9e8428c in 906ms, sequenceid=460, compaction requested=false 2024-11-21T03:25:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:18,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:25:18,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:18,826 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T03:25:18,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/fe5f794fcd7f41a3847961396bf88050 is 50, key is test_row_0/A:col10/1732159517912/Put/seqid=0 2024-11-21T03:25:18,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742167_1343 (size=12301) 2024-11-21T03:25:18,847 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/fe5f794fcd7f41a3847961396bf88050 2024-11-21T03:25:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/e99158ef50c847daa83e3ba7850632a6 is 50, key is test_row_0/B:col10/1732159517912/Put/seqid=0 2024-11-21T03:25:18,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742168_1344 (size=12301) 2024-11-21T03:25:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:19,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:19,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159579072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159579072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159579073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159579074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159579078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159579177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159579188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159579188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159579188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,296 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/e99158ef50c847daa83e3ba7850632a6 2024-11-21T03:25:19,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/93b4bcb155524f00aafa5b8da435bf37 is 50, key is test_row_0/C:col10/1732159517912/Put/seqid=0 2024-11-21T03:25:19,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742169_1345 (size=12301) 2024-11-21T03:25:19,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159579388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159579397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159579397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159579397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:25:19,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159579697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,709 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/93b4bcb155524f00aafa5b8da435bf37 2024-11-21T03:25:19,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159579707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159579707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159579708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:19,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/fe5f794fcd7f41a3847961396bf88050 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/fe5f794fcd7f41a3847961396bf88050 2024-11-21T03:25:19,716 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/fe5f794fcd7f41a3847961396bf88050, entries=150, sequenceid=484, filesize=12.0 K 2024-11-21T03:25:19,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/e99158ef50c847daa83e3ba7850632a6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e99158ef50c847daa83e3ba7850632a6 2024-11-21T03:25:19,720 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e99158ef50c847daa83e3ba7850632a6, entries=150, sequenceid=484, filesize=12.0 K 2024-11-21T03:25:19,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/93b4bcb155524f00aafa5b8da435bf37 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/93b4bcb155524f00aafa5b8da435bf37 2024-11-21T03:25:19,724 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/93b4bcb155524f00aafa5b8da435bf37, entries=150, sequenceid=484, filesize=12.0 K 2024-11-21T03:25:19,725 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6b132942b031ffcc7f2f51c4c9e8428c in 899ms, sequenceid=484, compaction requested=true 2024-11-21T03:25:19,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:19,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:19,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-21T03:25:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-21T03:25:19,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-21T03:25:19,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4580 sec 2024-11-21T03:25:19,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.4710 sec 2024-11-21T03:25:20,103 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:20,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T03:25:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:20,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/deda11cfd0474b6d8ead7b5dd8e374cc is 50, key is test_row_0/A:col10/1732159519064/Put/seqid=0 2024-11-21T03:25:20,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742170_1346 (size=12301) 2024-11-21T03:25:20,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/deda11cfd0474b6d8ead7b5dd8e374cc 2024-11-21T03:25:20,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/b4f7421bcb144aa9941d22a59fa19bcb is 50, key is test_row_0/B:col10/1732159519064/Put/seqid=0 2024-11-21T03:25:20,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742171_1347 (size=12301) 2024-11-21T03:25:20,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/b4f7421bcb144aa9941d22a59fa19bcb 2024-11-21T03:25:20,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/65e67088dc3a4ea3974306c5ee6e3c14 is 50, key is test_row_0/C:col10/1732159519064/Put/seqid=0 2024-11-21T03:25:20,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742172_1348 (size=12301) 2024-11-21T03:25:20,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159580217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159580221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159580260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159580260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159580260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159580368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159580377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159580377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159580578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159580588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159580588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/65e67088dc3a4ea3974306c5ee6e3c14 2024-11-21T03:25:20,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/deda11cfd0474b6d8ead7b5dd8e374cc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/deda11cfd0474b6d8ead7b5dd8e374cc 2024-11-21T03:25:20,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/deda11cfd0474b6d8ead7b5dd8e374cc, entries=150, sequenceid=500, filesize=12.0 K 2024-11-21T03:25:20,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/b4f7421bcb144aa9941d22a59fa19bcb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b4f7421bcb144aa9941d22a59fa19bcb 2024-11-21T03:25:20,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b4f7421bcb144aa9941d22a59fa19bcb, entries=150, sequenceid=500, filesize=12.0 K 2024-11-21T03:25:20,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/65e67088dc3a4ea3974306c5ee6e3c14 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65e67088dc3a4ea3974306c5ee6e3c14 2024-11-21T03:25:20,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65e67088dc3a4ea3974306c5ee6e3c14, entries=150, sequenceid=500, filesize=12.0 K 2024-11-21T03:25:20,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6b132942b031ffcc7f2f51c4c9e8428c in 553ms, sequenceid=500, compaction requested=true 2024-11-21T03:25:20,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:20,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:20,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:20,656 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:20,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:20,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:20,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:20,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:25:20,656 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:20,659 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:20,659 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:20,659 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:20,659 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/18d95da6e3444cfdb13eae88a9959c8c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/72f19f2f1d8b4013b05f4a83b56e57f4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e99158ef50c847daa83e3ba7850632a6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b4f7421bcb144aa9941d22a59fa19bcb] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=49.0 K 2024-11-21T03:25:20,660 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:20,660 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:20,660 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:20,660 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/cd4d28738e0141fe8e000153dbf24a57, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/833eca263c124929b72dca14ff057c9b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/fe5f794fcd7f41a3847961396bf88050, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/deda11cfd0474b6d8ead7b5dd8e374cc] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=49.0 K 2024-11-21T03:25:20,660 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 18d95da6e3444cfdb13eae88a9959c8c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732159516033 2024-11-21T03:25:20,660 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd4d28738e0141fe8e000153dbf24a57, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732159516033 2024-11-21T03:25:20,660 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 72f19f2f1d8b4013b05f4a83b56e57f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732159517225 2024-11-21T03:25:20,661 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 833eca263c124929b72dca14ff057c9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732159517225 2024-11-21T03:25:20,661 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e99158ef50c847daa83e3ba7850632a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732159517904 2024-11-21T03:25:20,661 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting b4f7421bcb144aa9941d22a59fa19bcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732159519064 2024-11-21T03:25:20,661 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe5f794fcd7f41a3847961396bf88050, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732159517904 2024-11-21T03:25:20,663 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting deda11cfd0474b6d8ead7b5dd8e374cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732159519064 2024-11-21T03:25:20,678 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:20,678 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:20,678 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/8fc1710b59354864ac910a751559d0c7 is 50, key is test_row_0/B:col10/1732159519064/Put/seqid=0 2024-11-21T03:25:20,678 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/f64cce6f3b8942a1a103bc746dd45cfe is 50, key is test_row_0/A:col10/1732159519064/Put/seqid=0 2024-11-21T03:25:20,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742173_1349 (size=13459) 2024-11-21T03:25:20,710 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/8fc1710b59354864ac910a751559d0c7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8fc1710b59354864ac910a751559d0c7 2024-11-21T03:25:20,721 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into 8fc1710b59354864ac910a751559d0c7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:20,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742174_1350 (size=13459) 2024-11-21T03:25:20,721 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:20,721 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=12, startTime=1732159520656; duration=0sec 2024-11-21T03:25:20,721 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:20,721 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:20,721 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:20,722 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:20,722 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:20,722 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:20,723 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/59beceb33d114356ade87de63551aee4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d575e584f2e4490589d0e649a8276e52, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/93b4bcb155524f00aafa5b8da435bf37, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65e67088dc3a4ea3974306c5ee6e3c14] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=49.0 K 2024-11-21T03:25:20,723 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 59beceb33d114356ade87de63551aee4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732159516033 2024-11-21T03:25:20,723 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d575e584f2e4490589d0e649a8276e52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732159517225 2024-11-21T03:25:20,724 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 93b4bcb155524f00aafa5b8da435bf37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=484, earliestPutTs=1732159517904 2024-11-21T03:25:20,724 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 65e67088dc3a4ea3974306c5ee6e3c14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732159519064 2024-11-21T03:25:20,744 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:20,745 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/13a969a372ce4ffbbee832bb5de56269 is 50, key is test_row_0/C:col10/1732159519064/Put/seqid=0 2024-11-21T03:25:20,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742175_1351 (size=13459) 2024-11-21T03:25:20,754 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/13a969a372ce4ffbbee832bb5de56269 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/13a969a372ce4ffbbee832bb5de56269 2024-11-21T03:25:20,768 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 13a969a372ce4ffbbee832bb5de56269(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:20,768 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:20,768 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=12, startTime=1732159520656; duration=0sec 2024-11-21T03:25:20,768 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:20,768 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:20,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T03:25:20,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/7c79341fa4544c05b97c966e682f7d6a is 50, key is test_row_0/A:col10/1732159520213/Put/seqid=0 2024-11-21T03:25:20,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742176_1352 (size=12301) 2024-11-21T03:25:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159580925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159580925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159580926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159581037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159581038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159581038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,129 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/f64cce6f3b8942a1a103bc746dd45cfe as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f64cce6f3b8942a1a103bc746dd45cfe 2024-11-21T03:25:21,137 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into f64cce6f3b8942a1a103bc746dd45cfe(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:21,137 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:21,137 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=12, startTime=1732159520656; duration=0sec 2024-11-21T03:25:21,137 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:21,137 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:21,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159581247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159581248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159581248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42778 deadline: 1732159581268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,269 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_1' in 6b132942b031ffcc7f2f51c4c9e8428c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:25:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42790 deadline: 1732159581268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,272 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_0' in 6b132942b031ffcc7f2f51c4c9e8428c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:25:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/7c79341fa4544c05b97c966e682f7d6a 2024-11-21T03:25:21,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/dc9ad1e13e834d34990e6795b6c89474 is 50, key is test_row_0/B:col10/1732159520213/Put/seqid=0 2024-11-21T03:25:21,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742177_1353 (size=12301) 2024-11-21T03:25:21,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/dc9ad1e13e834d34990e6795b6c89474 2024-11-21T03:25:21,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d3cdf501f20c477185997ab758aa7c32 is 50, key is test_row_0/C:col10/1732159520213/Put/seqid=0 2024-11-21T03:25:21,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742178_1354 (size=12301) 2024-11-21T03:25:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:25:21,469 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:21,472 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-21T03:25:21,475 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:21,476 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:21,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:25:21,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159581558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159581558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159581568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:25:21,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T03:25:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:21,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:25:21,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T03:25:21,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:21,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:21,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:21,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:21,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:21,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d3cdf501f20c477185997ab758aa7c32 2024-11-21T03:25:21,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/7c79341fa4544c05b97c966e682f7d6a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7c79341fa4544c05b97c966e682f7d6a 2024-11-21T03:25:21,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7c79341fa4544c05b97c966e682f7d6a, entries=150, sequenceid=523, filesize=12.0 K 2024-11-21T03:25:21,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/dc9ad1e13e834d34990e6795b6c89474 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dc9ad1e13e834d34990e6795b6c89474 2024-11-21T03:25:21,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dc9ad1e13e834d34990e6795b6c89474, entries=150, sequenceid=523, filesize=12.0 K 2024-11-21T03:25:21,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d3cdf501f20c477185997ab758aa7c32 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d3cdf501f20c477185997ab758aa7c32 2024-11-21T03:25:21,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d3cdf501f20c477185997ab758aa7c32, entries=150, sequenceid=523, filesize=12.0 K 2024-11-21T03:25:21,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6b132942b031ffcc7f2f51c4c9e8428c in 965ms, sequenceid=523, compaction requested=false 2024-11-21T03:25:21,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:21,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T03:25:21,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:21,961 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T03:25:21,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/7789abf14c4246c3908b1a2eeb9dab74 is 50, key is test_row_0/A:col10/1732159520924/Put/seqid=0 2024-11-21T03:25:21,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742179_1355 (size=12301) 2024-11-21T03:25:22,001 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/7789abf14c4246c3908b1a2eeb9dab74 2024-11-21T03:25:22,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/197802472e3b4821948167fd6b26a620 is 50, key is test_row_0/B:col10/1732159520924/Put/seqid=0 2024-11-21T03:25:22,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742180_1356 (size=12301) 2024-11-21T03:25:22,029 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/197802472e3b4821948167fd6b26a620 2024-11-21T03:25:22,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/551392247acf48d2af87202eb807a874 is 50, key is test_row_0/C:col10/1732159520924/Put/seqid=0 2024-11-21T03:25:22,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. as already flushing 2024-11-21T03:25:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:22,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742181_1357 (size=12301) 2024-11-21T03:25:22,096 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/551392247acf48d2af87202eb807a874 2024-11-21T03:25:22,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/7789abf14c4246c3908b1a2eeb9dab74 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7789abf14c4246c3908b1a2eeb9dab74 2024-11-21T03:25:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:25:22,111 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7789abf14c4246c3908b1a2eeb9dab74, entries=150, sequenceid=539, filesize=12.0 K 2024-11-21T03:25:22,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/197802472e3b4821948167fd6b26a620 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/197802472e3b4821948167fd6b26a620 2024-11-21T03:25:22,116 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/197802472e3b4821948167fd6b26a620, entries=150, sequenceid=539, filesize=12.0 K 2024-11-21T03:25:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/551392247acf48d2af87202eb807a874 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/551392247acf48d2af87202eb807a874 2024-11-21T03:25:22,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159582119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159582119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159582121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,129 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/551392247acf48d2af87202eb807a874, entries=150, sequenceid=539, filesize=12.0 K 2024-11-21T03:25:22,132 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6b132942b031ffcc7f2f51c4c9e8428c in 170ms, sequenceid=539, compaction requested=true 2024-11-21T03:25:22,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:22,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:22,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-21T03:25:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-21T03:25:22,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-21T03:25:22,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 659 msec 2024-11-21T03:25:22,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 671 msec 2024-11-21T03:25:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:22,228 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:22,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:25:22,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/32a64ed608c74b9ca06528539c33a17e is 50, key is test_row_0/A:col10/1732159522117/Put/seqid=0 2024-11-21T03:25:22,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742182_1358 (size=17181) 2024-11-21T03:25:22,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/32a64ed608c74b9ca06528539c33a17e 2024-11-21T03:25:22,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/221acd09f1aa42109e4d03833efc55d3 is 50, key is test_row_0/B:col10/1732159522117/Put/seqid=0 2024-11-21T03:25:22,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159582242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159582245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742183_1359 (size=12301) 2024-11-21T03:25:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159582247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,352 INFO [Thread-1158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1158. 2024-11-21T03:25:22,352 DEBUG [Thread-1158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:22,352 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:22,352 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:22,352 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:22,352 INFO [Thread-1160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1160. 2024-11-21T03:25:22,353 DEBUG [Thread-1160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:22,353 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:22,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:22,353 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:22,354 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-21T03:25:22,354 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:22,354 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:22,354 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:22,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:22,356 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-11-21T03:25:22,356 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:22,356 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:22,356 INFO [Thread-1156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1156. 2024-11-21T03:25:22,356 DEBUG [Thread-1156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:22,356 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:22,356 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:22,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:22,356 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:22,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:22,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159582357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159582357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159582357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159582568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159582568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159582568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:25:22,618 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:22,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/221acd09f1aa42109e4d03833efc55d3 2024-11-21T03:25:22,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/f0ee170ea7224eb4aed26b7861ebca03 is 50, key is test_row_0/C:col10/1732159522117/Put/seqid=0 2024-11-21T03:25:22,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742184_1360 (size=12301) 2024-11-21T03:25:22,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42774 deadline: 1732159582878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42736 deadline: 1732159582878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42802 deadline: 1732159582878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:25:23,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/f0ee170ea7224eb4aed26b7861ebca03 2024-11-21T03:25:23,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/32a64ed608c74b9ca06528539c33a17e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/32a64ed608c74b9ca06528539c33a17e 2024-11-21T03:25:23,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/32a64ed608c74b9ca06528539c33a17e, entries=250, sequenceid=562, filesize=16.8 K 2024-11-21T03:25:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/221acd09f1aa42109e4d03833efc55d3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/221acd09f1aa42109e4d03833efc55d3 2024-11-21T03:25:23,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/221acd09f1aa42109e4d03833efc55d3, entries=150, sequenceid=562, filesize=12.0 K 2024-11-21T03:25:23,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/f0ee170ea7224eb4aed26b7861ebca03 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0ee170ea7224eb4aed26b7861ebca03 2024-11-21T03:25:23,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0ee170ea7224eb4aed26b7861ebca03, entries=150, sequenceid=562, filesize=12.0 K 2024-11-21T03:25:23,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 6b132942b031ffcc7f2f51c4c9e8428c in 859ms, sequenceid=562, compaction requested=true 2024-11-21T03:25:23,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:23,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:23,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:23,087 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:23,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:23,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:23,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6b132942b031ffcc7f2f51c4c9e8428c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:23,088 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:23,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:23,088 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:23,088 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:23,088 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/A is initiating minor compaction (all files) 2024-11-21T03:25:23,088 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/B is initiating minor compaction (all files) 2024-11-21T03:25:23,089 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/B in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:23,089 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/A in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:23,089 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8fc1710b59354864ac910a751559d0c7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dc9ad1e13e834d34990e6795b6c89474, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/197802472e3b4821948167fd6b26a620, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/221acd09f1aa42109e4d03833efc55d3] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=49.2 K 2024-11-21T03:25:23,089 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f64cce6f3b8942a1a103bc746dd45cfe, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7c79341fa4544c05b97c966e682f7d6a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7789abf14c4246c3908b1a2eeb9dab74, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/32a64ed608c74b9ca06528539c33a17e] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=53.9 K 2024-11-21T03:25:23,089 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc1710b59354864ac910a751559d0c7, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732159519064 2024-11-21T03:25:23,089 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f64cce6f3b8942a1a103bc746dd45cfe, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732159519064 2024-11-21T03:25:23,089 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c79341fa4544c05b97c966e682f7d6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732159520213 2024-11-21T03:25:23,089 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting dc9ad1e13e834d34990e6795b6c89474, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732159520213 2024-11-21T03:25:23,089 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7789abf14c4246c3908b1a2eeb9dab74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1732159520916 2024-11-21T03:25:23,089 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 197802472e3b4821948167fd6b26a620, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1732159520916 2024-11-21T03:25:23,089 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a64ed608c74b9ca06528539c33a17e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732159522117 2024-11-21T03:25:23,090 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 221acd09f1aa42109e4d03833efc55d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732159522117 2024-11-21T03:25:23,097 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:23,098 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/465afe1cf6714b339f4f884148527d7f is 50, key is test_row_0/B:col10/1732159522117/Put/seqid=0 2024-11-21T03:25:23,104 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#A#compaction#313 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:23,105 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/b0e77e2362724bb4b3542999e67e3421 is 50, key is test_row_0/A:col10/1732159522117/Put/seqid=0 2024-11-21T03:25:23,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742185_1361 (size=13595) 2024-11-21T03:25:23,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742186_1362 (size=13595) 2024-11-21T03:25:23,113 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/b0e77e2362724bb4b3542999e67e3421 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/b0e77e2362724bb4b3542999e67e3421 2024-11-21T03:25:23,117 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/A of 6b132942b031ffcc7f2f51c4c9e8428c into b0e77e2362724bb4b3542999e67e3421(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:23,117 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:23,117 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/A, priority=12, startTime=1732159523087; duration=0sec 2024-11-21T03:25:23,118 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:23,118 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:A 2024-11-21T03:25:23,118 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:23,119 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:23,119 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 6b132942b031ffcc7f2f51c4c9e8428c/C is initiating minor compaction (all files) 2024-11-21T03:25:23,119 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6b132942b031ffcc7f2f51c4c9e8428c/C in TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:23,119 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/13a969a372ce4ffbbee832bb5de56269, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d3cdf501f20c477185997ab758aa7c32, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/551392247acf48d2af87202eb807a874, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0ee170ea7224eb4aed26b7861ebca03] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp, totalSize=49.2 K 2024-11-21T03:25:23,119 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13a969a372ce4ffbbee832bb5de56269, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=500, earliestPutTs=1732159519064 2024-11-21T03:25:23,120 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3cdf501f20c477185997ab758aa7c32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1732159520213 2024-11-21T03:25:23,120 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 551392247acf48d2af87202eb807a874, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1732159520916 2024-11-21T03:25:23,120 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0ee170ea7224eb4aed26b7861ebca03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1732159522117 2024-11-21T03:25:23,130 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6b132942b031ffcc7f2f51c4c9e8428c#C#compaction#314 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:23,131 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/8a029f42ba5549f684fbfc040141ef8a is 50, key is test_row_0/C:col10/1732159522117/Put/seqid=0 2024-11-21T03:25:23,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742187_1363 (size=13595) 2024-11-21T03:25:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:23,290 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:23,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:25:23,290 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-21T03:25:23,290 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:23,290 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,291 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:23,307 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-21T03:25:23,307 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:23,307 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,307 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:23,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/48af540588aa41b58d1091b2b0a257ae is 50, key is test_row_0/A:col10/1732159522232/Put/seqid=0 2024-11-21T03:25:23,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742188_1364 (size=12301) 2024-11-21T03:25:23,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/48af540588aa41b58d1091b2b0a257ae 2024-11-21T03:25:23,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/abfbf3ed994248c5bd2cdc502d1c596f is 50, key is test_row_0/B:col10/1732159522232/Put/seqid=0 2024-11-21T03:25:23,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742189_1365 (size=12301) 2024-11-21T03:25:23,390 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-21T03:25:23,390 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-21T03:25:23,390 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-11-21T03:25:23,390 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:23,390 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:23,390 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,390 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:23,390 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,390 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,390 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,390 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,390 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:23,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:23,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T03:25:23,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-21T03:25:23,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-21T03:25:23,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-21T03:25:23,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-21T03:25:23,390 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-21T03:25:23,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T03:25:23,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2927 2024-11-21T03:25:23,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2845 2024-11-21T03:25:23,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2860 2024-11-21T03:25:23,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2931 2024-11-21T03:25:23,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2868 2024-11-21T03:25:23,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T03:25:23,391 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:25:23,391 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:25:23,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:23,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:23,392 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T03:25:23,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:25:23,396 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159523396"}]},"ts":"1732159523396"} 2024-11-21T03:25:23,400 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T03:25:23,400 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T03:25:23,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:25:23,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b132942b031ffcc7f2f51c4c9e8428c, UNASSIGN}] 2024-11-21T03:25:23,406 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b132942b031ffcc7f2f51c4c9e8428c, UNASSIGN 2024-11-21T03:25:23,407 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=6b132942b031ffcc7f2f51c4c9e8428c, regionState=CLOSING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:23,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b132942b031ffcc7f2f51c4c9e8428c, UNASSIGN because future has completed 2024-11-21T03:25:23,408 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:25:23,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:25:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:25:23,512 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/465afe1cf6714b339f4f884148527d7f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/465afe1cf6714b339f4f884148527d7f 2024-11-21T03:25:23,518 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/B of 6b132942b031ffcc7f2f51c4c9e8428c into 465afe1cf6714b339f4f884148527d7f(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:23,518 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:23,518 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/B, priority=12, startTime=1732159523087; duration=0sec 2024-11-21T03:25:23,518 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:23,518 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:B 2024-11-21T03:25:23,540 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/8a029f42ba5549f684fbfc040141ef8a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/8a029f42ba5549f684fbfc040141ef8a 2024-11-21T03:25:23,544 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6b132942b031ffcc7f2f51c4c9e8428c/C of 6b132942b031ffcc7f2f51c4c9e8428c into 8a029f42ba5549f684fbfc040141ef8a(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:23,544 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:23,544 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., storeName=6b132942b031ffcc7f2f51c4c9e8428c/C, priority=12, startTime=1732159523088; duration=0sec 2024-11-21T03:25:23,544 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:23,544 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6b132942b031ffcc7f2f51c4c9e8428c:C 2024-11-21T03:25:23,563 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:23,563 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:25:23,563 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing 6b132942b031ffcc7f2f51c4c9e8428c, disabling compactions & flushes 2024-11-21T03:25:23,563 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:25:23,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/abfbf3ed994248c5bd2cdc502d1c596f 2024-11-21T03:25:23,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/35ae3d21862c433384af79cd2bcf57db is 50, key is test_row_0/C:col10/1732159522232/Put/seqid=0 2024-11-21T03:25:23,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742190_1366 (size=12301) 2024-11-21T03:25:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:25:24,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/35ae3d21862c433384af79cd2bcf57db 2024-11-21T03:25:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/48af540588aa41b58d1091b2b0a257ae as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/48af540588aa41b58d1091b2b0a257ae 2024-11-21T03:25:24,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/48af540588aa41b58d1091b2b0a257ae, entries=150, sequenceid=577, filesize=12.0 K 2024-11-21T03:25:24,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/abfbf3ed994248c5bd2cdc502d1c596f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/abfbf3ed994248c5bd2cdc502d1c596f 2024-11-21T03:25:24,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/abfbf3ed994248c5bd2cdc502d1c596f, entries=150, sequenceid=577, filesize=12.0 K 2024-11-21T03:25:24,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/35ae3d21862c433384af79cd2bcf57db as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/35ae3d21862c433384af79cd2bcf57db 2024-11-21T03:25:24,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/35ae3d21862c433384af79cd2bcf57db, entries=150, sequenceid=577, filesize=12.0 K 2024-11-21T03:25:24,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=26.84 KB/27480 for 6b132942b031ffcc7f2f51c4c9e8428c in 891ms, sequenceid=577, compaction requested=false 2024-11-21T03:25:24,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6b132942b031ffcc7f2f51c4c9e8428c: 2024-11-21T03:25:24,181 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:24,181 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:24,181 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. after waiting 0 ms 2024-11-21T03:25:24,181 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:24,182 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing 6b132942b031ffcc7f2f51c4c9e8428c 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-21T03:25:24,186 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/9f3cf08671e44d7c8d99d7651df0e44a is 50, key is test_row_0/A:col10/1732159523388/Put/seqid=0 2024-11-21T03:25:24,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742191_1367 (size=12301) 2024-11-21T03:25:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:25:24,607 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/9f3cf08671e44d7c8d99d7651df0e44a 2024-11-21T03:25:24,614 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/eda27034fb5b402ebcbfe972edc15b1b is 50, key is test_row_0/B:col10/1732159523388/Put/seqid=0 2024-11-21T03:25:24,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742192_1368 (size=12301) 2024-11-21T03:25:25,028 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/eda27034fb5b402ebcbfe972edc15b1b 2024-11-21T03:25:25,053 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d2f24b4776d04a3fb3dedb8fea6491fd is 50, key is test_row_0/C:col10/1732159523388/Put/seqid=0 2024-11-21T03:25:25,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742193_1369 (size=12301) 2024-11-21T03:25:25,490 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d2f24b4776d04a3fb3dedb8fea6491fd 2024-11-21T03:25:25,495 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/A/9f3cf08671e44d7c8d99d7651df0e44a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/9f3cf08671e44d7c8d99d7651df0e44a 2024-11-21T03:25:25,514 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/9f3cf08671e44d7c8d99d7651df0e44a, entries=150, sequenceid=586, filesize=12.0 K 2024-11-21T03:25:25,515 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/B/eda27034fb5b402ebcbfe972edc15b1b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/eda27034fb5b402ebcbfe972edc15b1b 2024-11-21T03:25:25,522 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/eda27034fb5b402ebcbfe972edc15b1b, entries=150, sequenceid=586, filesize=12.0 K 2024-11-21T03:25:25,525 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/.tmp/C/d2f24b4776d04a3fb3dedb8fea6491fd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d2f24b4776d04a3fb3dedb8fea6491fd 2024-11-21T03:25:25,531 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d2f24b4776d04a3fb3dedb8fea6491fd, entries=150, sequenceid=586, filesize=12.0 K 2024-11-21T03:25:25,533 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6b132942b031ffcc7f2f51c4c9e8428c in 1352ms, sequenceid=586, compaction requested=true 2024-11-21T03:25:25,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f719985fe60846628dab220cd2bf718b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c34959e2356d4ea598755384b7f53738, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e122ea5a791647fca67fc16feb8dab13, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8ab32959f4f7420d9a57ab7822aeccdb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d79edddc43f54780aa64241e9148cf19, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/77c85c605f8048859e44bdbf3fdff50f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5dd496c7f2064e2a9fbd32dbea8561be, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/837e0ac303e34f4c918f9287c1435faa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/65fb1739bcd74206a795b3c2a5538967, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c555bfcfefdf4f87bec932644e979062, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/788bfe861d1946bfbcfca72df6035754, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/867bab1cad104d0da6580ec5270e7b64, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f52ecaa049fd409490565c5887e9d60e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/0cac5dc274724286a52a4286a6f929f4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4420da4d37734de1801d2e4d48d4ccc9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/19e42dc476ff4e38949216d31ab55cf3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8d9aef9f20aa4f61a0895e5f767e2256, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d0127504cf3847b7aa2e26152059ea7d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4d0184ba789e4f9ea1d30aaf75b360e5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/bd09f66802074f809c9e777c45e61a5a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5336aca212a04c13b589dec6c636e7aa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/df767503554340049ce5c54fe958f592, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/57f0cdd900c6411589cf7062eac8bae7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/125c8e2b90ba4bb0ad10b8d8afdd11af, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/ffc5bf2b5fa340298165e7266efa21ae, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e58f22b61e0b4e978bf3b5a7697ab26c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/daddb7f4b13849ed999b44ea06f3d98e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/12f098386bdc4bcc924f984aef6b99eb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5d7386e2863e48ee8a660a7831027a60, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e097524deefa444fa0bd7ab858e8367c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/cd4d28738e0141fe8e000153dbf24a57, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/833eca263c124929b72dca14ff057c9b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/fe5f794fcd7f41a3847961396bf88050, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f64cce6f3b8942a1a103bc746dd45cfe, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/deda11cfd0474b6d8ead7b5dd8e374cc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7c79341fa4544c05b97c966e682f7d6a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7789abf14c4246c3908b1a2eeb9dab74, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/32a64ed608c74b9ca06528539c33a17e] to archive 2024-11-21T03:25:25,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:25:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:25:25,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f719985fe60846628dab220cd2bf718b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f719985fe60846628dab220cd2bf718b 2024-11-21T03:25:25,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c34959e2356d4ea598755384b7f53738 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c34959e2356d4ea598755384b7f53738 2024-11-21T03:25:25,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e122ea5a791647fca67fc16feb8dab13 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e122ea5a791647fca67fc16feb8dab13 2024-11-21T03:25:25,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8ab32959f4f7420d9a57ab7822aeccdb to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8ab32959f4f7420d9a57ab7822aeccdb 2024-11-21T03:25:25,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d79edddc43f54780aa64241e9148cf19 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d79edddc43f54780aa64241e9148cf19 2024-11-21T03:25:25,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/77c85c605f8048859e44bdbf3fdff50f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/77c85c605f8048859e44bdbf3fdff50f 2024-11-21T03:25:25,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5dd496c7f2064e2a9fbd32dbea8561be to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5dd496c7f2064e2a9fbd32dbea8561be 2024-11-21T03:25:25,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/837e0ac303e34f4c918f9287c1435faa to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/837e0ac303e34f4c918f9287c1435faa 2024-11-21T03:25:25,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/65fb1739bcd74206a795b3c2a5538967 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/65fb1739bcd74206a795b3c2a5538967 2024-11-21T03:25:25,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c555bfcfefdf4f87bec932644e979062 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/c555bfcfefdf4f87bec932644e979062 2024-11-21T03:25:25,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/788bfe861d1946bfbcfca72df6035754 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/788bfe861d1946bfbcfca72df6035754 2024-11-21T03:25:25,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/867bab1cad104d0da6580ec5270e7b64 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/867bab1cad104d0da6580ec5270e7b64 2024-11-21T03:25:25,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f52ecaa049fd409490565c5887e9d60e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f52ecaa049fd409490565c5887e9d60e 2024-11-21T03:25:25,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/0cac5dc274724286a52a4286a6f929f4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/0cac5dc274724286a52a4286a6f929f4 2024-11-21T03:25:25,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4420da4d37734de1801d2e4d48d4ccc9 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4420da4d37734de1801d2e4d48d4ccc9 2024-11-21T03:25:25,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/19e42dc476ff4e38949216d31ab55cf3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/19e42dc476ff4e38949216d31ab55cf3 2024-11-21T03:25:25,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8d9aef9f20aa4f61a0895e5f767e2256 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/8d9aef9f20aa4f61a0895e5f767e2256 2024-11-21T03:25:25,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d0127504cf3847b7aa2e26152059ea7d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/d0127504cf3847b7aa2e26152059ea7d 2024-11-21T03:25:25,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4d0184ba789e4f9ea1d30aaf75b360e5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/4d0184ba789e4f9ea1d30aaf75b360e5 2024-11-21T03:25:25,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/bd09f66802074f809c9e777c45e61a5a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/bd09f66802074f809c9e777c45e61a5a 2024-11-21T03:25:25,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5336aca212a04c13b589dec6c636e7aa to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5336aca212a04c13b589dec6c636e7aa 2024-11-21T03:25:25,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/df767503554340049ce5c54fe958f592 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/df767503554340049ce5c54fe958f592 2024-11-21T03:25:25,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/57f0cdd900c6411589cf7062eac8bae7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/57f0cdd900c6411589cf7062eac8bae7 2024-11-21T03:25:25,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/125c8e2b90ba4bb0ad10b8d8afdd11af to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/125c8e2b90ba4bb0ad10b8d8afdd11af 2024-11-21T03:25:25,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/ffc5bf2b5fa340298165e7266efa21ae to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/ffc5bf2b5fa340298165e7266efa21ae 2024-11-21T03:25:25,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e58f22b61e0b4e978bf3b5a7697ab26c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e58f22b61e0b4e978bf3b5a7697ab26c 2024-11-21T03:25:25,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/daddb7f4b13849ed999b44ea06f3d98e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/daddb7f4b13849ed999b44ea06f3d98e 2024-11-21T03:25:25,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/12f098386bdc4bcc924f984aef6b99eb to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/12f098386bdc4bcc924f984aef6b99eb 2024-11-21T03:25:25,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5d7386e2863e48ee8a660a7831027a60 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/5d7386e2863e48ee8a660a7831027a60 2024-11-21T03:25:25,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e097524deefa444fa0bd7ab858e8367c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/e097524deefa444fa0bd7ab858e8367c 2024-11-21T03:25:25,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/cd4d28738e0141fe8e000153dbf24a57 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/cd4d28738e0141fe8e000153dbf24a57 2024-11-21T03:25:25,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/833eca263c124929b72dca14ff057c9b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/833eca263c124929b72dca14ff057c9b 2024-11-21T03:25:25,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/fe5f794fcd7f41a3847961396bf88050 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/fe5f794fcd7f41a3847961396bf88050 2024-11-21T03:25:25,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f64cce6f3b8942a1a103bc746dd45cfe to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/f64cce6f3b8942a1a103bc746dd45cfe 2024-11-21T03:25:25,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/deda11cfd0474b6d8ead7b5dd8e374cc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/deda11cfd0474b6d8ead7b5dd8e374cc 2024-11-21T03:25:25,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7c79341fa4544c05b97c966e682f7d6a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7c79341fa4544c05b97c966e682f7d6a 2024-11-21T03:25:25,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7789abf14c4246c3908b1a2eeb9dab74 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/7789abf14c4246c3908b1a2eeb9dab74 2024-11-21T03:25:25,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/32a64ed608c74b9ca06528539c33a17e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/32a64ed608c74b9ca06528539c33a17e 2024-11-21T03:25:25,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/9e0a4ad91f284bfc8837cbe38b69501f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/140f2d8f35734cfcb06a94a3c8d15a25, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f4eff044f17d4d3aa8b1cbaa19bc4b35, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f6ab0304a1c34fe38d19f2d0348a4980, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e7f3df9615e4414e9c81b90c9f86b00b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c1c318920c6d41caa3e68b6174a1506f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/113af1cc13834e648e1ed4c96b91ccd3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/cca0641a0db8432b91a634bc66f14289, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8ad66e183db1471898f05da76debc4b6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7c83412a47354a15af78da939c195e5a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dea1d3c9d4f646e494865f5678ebcc51, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/4e8d5e74170942d0848606bfd05dff6f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/2afc9bfd7e5343a0aba161ab5b50a47e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/092aac1a2b3a49e7bc6f5fcfee7ae3f3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/85d5635031e94c7387e073088de6495f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/90083ccac6074dbfa6017ef8811825ce, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/84cb98249a564d419e7490dbf54e4595, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/59f7523eb19947ad906818f5dbbd64f4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c2c97a405703441d86ef055147dbf794, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/537bf96093984986a2a0dedc4c2857c2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/370dd6fbebef469295026b5ac09bcbae, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7e28559a71044078b7b4794c9f745070, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/993def98ee6842daac72fc7f6cad49b6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a98d1dc9c8d41d08c09f8d27db0f183, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/ce4f69cef4cd4350802c312b6a34be18, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b1c852aaf15342809a805a2d661f576f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/46c4f2f43eab40af90d519c3d4fa7fd3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/6cc64f90558b423d81e0585bb6d3a79d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/800b44dbda184d02b89d71719d399d98, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/18d95da6e3444cfdb13eae88a9959c8c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a8cbe4f21d2439689a5f7bbac041cb3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/72f19f2f1d8b4013b05f4a83b56e57f4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e99158ef50c847daa83e3ba7850632a6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8fc1710b59354864ac910a751559d0c7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b4f7421bcb144aa9941d22a59fa19bcb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dc9ad1e13e834d34990e6795b6c89474, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/197802472e3b4821948167fd6b26a620, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/221acd09f1aa42109e4d03833efc55d3] to archive 2024-11-21T03:25:25,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:25:25,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/9e0a4ad91f284bfc8837cbe38b69501f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/9e0a4ad91f284bfc8837cbe38b69501f 2024-11-21T03:25:25,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/140f2d8f35734cfcb06a94a3c8d15a25 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/140f2d8f35734cfcb06a94a3c8d15a25 2024-11-21T03:25:25,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f4eff044f17d4d3aa8b1cbaa19bc4b35 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f4eff044f17d4d3aa8b1cbaa19bc4b35 2024-11-21T03:25:25,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f6ab0304a1c34fe38d19f2d0348a4980 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/f6ab0304a1c34fe38d19f2d0348a4980 2024-11-21T03:25:25,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e7f3df9615e4414e9c81b90c9f86b00b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e7f3df9615e4414e9c81b90c9f86b00b 2024-11-21T03:25:25,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c1c318920c6d41caa3e68b6174a1506f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c1c318920c6d41caa3e68b6174a1506f 2024-11-21T03:25:25,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/113af1cc13834e648e1ed4c96b91ccd3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/113af1cc13834e648e1ed4c96b91ccd3 2024-11-21T03:25:25,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/cca0641a0db8432b91a634bc66f14289 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/cca0641a0db8432b91a634bc66f14289 2024-11-21T03:25:25,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8ad66e183db1471898f05da76debc4b6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8ad66e183db1471898f05da76debc4b6 2024-11-21T03:25:25,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7c83412a47354a15af78da939c195e5a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7c83412a47354a15af78da939c195e5a 2024-11-21T03:25:25,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dea1d3c9d4f646e494865f5678ebcc51 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dea1d3c9d4f646e494865f5678ebcc51 2024-11-21T03:25:25,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/4e8d5e74170942d0848606bfd05dff6f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/4e8d5e74170942d0848606bfd05dff6f 2024-11-21T03:25:25,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/2afc9bfd7e5343a0aba161ab5b50a47e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/2afc9bfd7e5343a0aba161ab5b50a47e 2024-11-21T03:25:25,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/092aac1a2b3a49e7bc6f5fcfee7ae3f3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/092aac1a2b3a49e7bc6f5fcfee7ae3f3 2024-11-21T03:25:25,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/85d5635031e94c7387e073088de6495f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/85d5635031e94c7387e073088de6495f 2024-11-21T03:25:25,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/90083ccac6074dbfa6017ef8811825ce to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/90083ccac6074dbfa6017ef8811825ce 2024-11-21T03:25:25,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/84cb98249a564d419e7490dbf54e4595 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/84cb98249a564d419e7490dbf54e4595 2024-11-21T03:25:25,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/59f7523eb19947ad906818f5dbbd64f4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/59f7523eb19947ad906818f5dbbd64f4 2024-11-21T03:25:25,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c2c97a405703441d86ef055147dbf794 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/c2c97a405703441d86ef055147dbf794 2024-11-21T03:25:25,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/537bf96093984986a2a0dedc4c2857c2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/537bf96093984986a2a0dedc4c2857c2 2024-11-21T03:25:25,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/370dd6fbebef469295026b5ac09bcbae to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/370dd6fbebef469295026b5ac09bcbae 2024-11-21T03:25:25,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7e28559a71044078b7b4794c9f745070 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/7e28559a71044078b7b4794c9f745070 2024-11-21T03:25:25,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/993def98ee6842daac72fc7f6cad49b6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/993def98ee6842daac72fc7f6cad49b6 2024-11-21T03:25:25,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a98d1dc9c8d41d08c09f8d27db0f183 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a98d1dc9c8d41d08c09f8d27db0f183 2024-11-21T03:25:25,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/ce4f69cef4cd4350802c312b6a34be18 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/ce4f69cef4cd4350802c312b6a34be18 2024-11-21T03:25:25,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b1c852aaf15342809a805a2d661f576f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b1c852aaf15342809a805a2d661f576f 2024-11-21T03:25:25,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/46c4f2f43eab40af90d519c3d4fa7fd3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/46c4f2f43eab40af90d519c3d4fa7fd3 2024-11-21T03:25:25,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/6cc64f90558b423d81e0585bb6d3a79d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/6cc64f90558b423d81e0585bb6d3a79d 2024-11-21T03:25:25,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/800b44dbda184d02b89d71719d399d98 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/800b44dbda184d02b89d71719d399d98 2024-11-21T03:25:25,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/18d95da6e3444cfdb13eae88a9959c8c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/18d95da6e3444cfdb13eae88a9959c8c 2024-11-21T03:25:25,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a8cbe4f21d2439689a5f7bbac041cb3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/1a8cbe4f21d2439689a5f7bbac041cb3 2024-11-21T03:25:25,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/72f19f2f1d8b4013b05f4a83b56e57f4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/72f19f2f1d8b4013b05f4a83b56e57f4 2024-11-21T03:25:25,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e99158ef50c847daa83e3ba7850632a6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/e99158ef50c847daa83e3ba7850632a6 2024-11-21T03:25:25,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8fc1710b59354864ac910a751559d0c7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/8fc1710b59354864ac910a751559d0c7 2024-11-21T03:25:25,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b4f7421bcb144aa9941d22a59fa19bcb to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/b4f7421bcb144aa9941d22a59fa19bcb 2024-11-21T03:25:25,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dc9ad1e13e834d34990e6795b6c89474 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/dc9ad1e13e834d34990e6795b6c89474 2024-11-21T03:25:25,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/197802472e3b4821948167fd6b26a620 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/197802472e3b4821948167fd6b26a620 2024-11-21T03:25:25,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/221acd09f1aa42109e4d03833efc55d3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/221acd09f1aa42109e4d03833efc55d3 2024-11-21T03:25:25,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a966dcd8580400aadef2656b0ea064d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e485fe82194c4763ade81a50a75a187f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/78b80e5625994796a53cb9b6db5c9831, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e970478390e44505aa422980350f0f0d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ed6e8937c0ae4fdfbf206142345c97d6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/69166415d8404226beff221db57c0a5c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b28104ee4bc4f57b37541895fc44866, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/71f0f747178d49dcaf0f69ab563ca8a3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/861f3c1c9354456ab63e9fb6363219a7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/25b6a047a3474f08b49dd76f4c54dea7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/54fbe3b158c64cfb8e513cefce82c95e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0f0a05571e549d298c7bd6b8035ed75, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/995c00fb7b1c42a0af737c6ca7aaa29d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/8723dac80c3441db8d9f10dd90a5ca08, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65860daa1a9e4dfca52f127ccd18a0e8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/03c0f68f26414e7fa0fcf193014e45b3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a041152f36b4c0fac0b67575e7251ff, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ebeaf54616234b42abce6199e2bf496e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9b2a7cfefaca4a80823ed4423c8b11ef, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9e178e1302b945cfadd971089a2756a0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/5533a71e0046409abbbf211e946c0c5b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d62245deda2848abb03ea2ac76c8808d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0c90b150d2ec419380192780ee71ae67, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/eb2755e1df3747ac8596770b0484c32e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/7b8cb2f515634405848aa419d3a0936c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0e63b88b00d743be8df9f836a9836f18, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/908c832d0693484db32695b2cde74729, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b6490b8af1c479f8d4a08b2e5625d16, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d5a45024ae8f4332a9e901ac436cd7be, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/59beceb33d114356ade87de63551aee4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/cc98ccaf17ea421a854179d44b777b67, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d575e584f2e4490589d0e649a8276e52, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/93b4bcb155524f00aafa5b8da435bf37, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/13a969a372ce4ffbbee832bb5de56269, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65e67088dc3a4ea3974306c5ee6e3c14, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d3cdf501f20c477185997ab758aa7c32, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/551392247acf48d2af87202eb807a874, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0ee170ea7224eb4aed26b7861ebca03] to archive 2024-11-21T03:25:25,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:25:25,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a966dcd8580400aadef2656b0ea064d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a966dcd8580400aadef2656b0ea064d 2024-11-21T03:25:25,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e485fe82194c4763ade81a50a75a187f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e485fe82194c4763ade81a50a75a187f 2024-11-21T03:25:25,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/78b80e5625994796a53cb9b6db5c9831 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/78b80e5625994796a53cb9b6db5c9831 2024-11-21T03:25:25,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e970478390e44505aa422980350f0f0d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/e970478390e44505aa422980350f0f0d 2024-11-21T03:25:25,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ed6e8937c0ae4fdfbf206142345c97d6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ed6e8937c0ae4fdfbf206142345c97d6 2024-11-21T03:25:25,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/69166415d8404226beff221db57c0a5c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/69166415d8404226beff221db57c0a5c 2024-11-21T03:25:25,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b28104ee4bc4f57b37541895fc44866 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b28104ee4bc4f57b37541895fc44866 2024-11-21T03:25:25,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/71f0f747178d49dcaf0f69ab563ca8a3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/71f0f747178d49dcaf0f69ab563ca8a3 2024-11-21T03:25:25,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/861f3c1c9354456ab63e9fb6363219a7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/861f3c1c9354456ab63e9fb6363219a7 2024-11-21T03:25:25,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/25b6a047a3474f08b49dd76f4c54dea7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/25b6a047a3474f08b49dd76f4c54dea7 2024-11-21T03:25:25,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/54fbe3b158c64cfb8e513cefce82c95e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/54fbe3b158c64cfb8e513cefce82c95e 2024-11-21T03:25:25,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0f0a05571e549d298c7bd6b8035ed75 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0f0a05571e549d298c7bd6b8035ed75 2024-11-21T03:25:25,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/995c00fb7b1c42a0af737c6ca7aaa29d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/995c00fb7b1c42a0af737c6ca7aaa29d 2024-11-21T03:25:25,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/8723dac80c3441db8d9f10dd90a5ca08 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/8723dac80c3441db8d9f10dd90a5ca08 2024-11-21T03:25:25,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65860daa1a9e4dfca52f127ccd18a0e8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65860daa1a9e4dfca52f127ccd18a0e8 2024-11-21T03:25:25,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/03c0f68f26414e7fa0fcf193014e45b3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/03c0f68f26414e7fa0fcf193014e45b3 2024-11-21T03:25:25,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a041152f36b4c0fac0b67575e7251ff to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0a041152f36b4c0fac0b67575e7251ff 2024-11-21T03:25:25,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ebeaf54616234b42abce6199e2bf496e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/ebeaf54616234b42abce6199e2bf496e 2024-11-21T03:25:25,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9b2a7cfefaca4a80823ed4423c8b11ef to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9b2a7cfefaca4a80823ed4423c8b11ef 2024-11-21T03:25:25,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9e178e1302b945cfadd971089a2756a0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/9e178e1302b945cfadd971089a2756a0 2024-11-21T03:25:25,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/5533a71e0046409abbbf211e946c0c5b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/5533a71e0046409abbbf211e946c0c5b 2024-11-21T03:25:25,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d62245deda2848abb03ea2ac76c8808d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d62245deda2848abb03ea2ac76c8808d 2024-11-21T03:25:25,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0c90b150d2ec419380192780ee71ae67 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0c90b150d2ec419380192780ee71ae67 2024-11-21T03:25:25,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/eb2755e1df3747ac8596770b0484c32e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/eb2755e1df3747ac8596770b0484c32e 2024-11-21T03:25:25,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/7b8cb2f515634405848aa419d3a0936c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/7b8cb2f515634405848aa419d3a0936c 2024-11-21T03:25:25,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0e63b88b00d743be8df9f836a9836f18 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/0e63b88b00d743be8df9f836a9836f18 2024-11-21T03:25:25,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/908c832d0693484db32695b2cde74729 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/908c832d0693484db32695b2cde74729 2024-11-21T03:25:25,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b6490b8af1c479f8d4a08b2e5625d16 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/2b6490b8af1c479f8d4a08b2e5625d16 2024-11-21T03:25:25,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d5a45024ae8f4332a9e901ac436cd7be to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d5a45024ae8f4332a9e901ac436cd7be 2024-11-21T03:25:25,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/59beceb33d114356ade87de63551aee4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/59beceb33d114356ade87de63551aee4 2024-11-21T03:25:25,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/cc98ccaf17ea421a854179d44b777b67 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/cc98ccaf17ea421a854179d44b777b67 2024-11-21T03:25:25,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d575e584f2e4490589d0e649a8276e52 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d575e584f2e4490589d0e649a8276e52 2024-11-21T03:25:25,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/93b4bcb155524f00aafa5b8da435bf37 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/93b4bcb155524f00aafa5b8da435bf37 2024-11-21T03:25:25,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/13a969a372ce4ffbbee832bb5de56269 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/13a969a372ce4ffbbee832bb5de56269 2024-11-21T03:25:25,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65e67088dc3a4ea3974306c5ee6e3c14 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/65e67088dc3a4ea3974306c5ee6e3c14 2024-11-21T03:25:25,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d3cdf501f20c477185997ab758aa7c32 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d3cdf501f20c477185997ab758aa7c32 2024-11-21T03:25:25,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/551392247acf48d2af87202eb807a874 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/551392247acf48d2af87202eb807a874 2024-11-21T03:25:25,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0ee170ea7224eb4aed26b7861ebca03 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/f0ee170ea7224eb4aed26b7861ebca03 2024-11-21T03:25:25,773 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/recovered.edits/589.seqid, newMaxSeqId=589, maxSeqId=1 2024-11-21T03:25:25,774 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. 2024-11-21T03:25:25,774 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for 6b132942b031ffcc7f2f51c4c9e8428c: Waiting for close lock at 1732159523563Running coprocessor pre-close hooks at 1732159523563Disabling compacts and flushes for region at 1732159523563Disabling writes for close at 1732159524181 (+618 ms)Obtaining lock to block concurrent updates at 1732159524182 (+1 ms)Preparing flush snapshotting stores in 6b132942b031ffcc7f2f51c4c9e8428c at 1732159524182Finished memstore snapshotting TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732159524182Flushing stores of TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c. at 1732159524182Flushing 6b132942b031ffcc7f2f51c4c9e8428c/A: creating writer at 1732159524182Flushing 6b132942b031ffcc7f2f51c4c9e8428c/A: appending metadata at 1732159524185 (+3 ms)Flushing 6b132942b031ffcc7f2f51c4c9e8428c/A: closing flushed file at 1732159524186 (+1 ms)Flushing 6b132942b031ffcc7f2f51c4c9e8428c/B: creating writer at 1732159524611 (+425 ms)Flushing 6b132942b031ffcc7f2f51c4c9e8428c/B: appending metadata at 1732159524614 (+3 ms)Flushing 6b132942b031ffcc7f2f51c4c9e8428c/B: closing flushed file at 1732159524614Flushing 6b132942b031ffcc7f2f51c4c9e8428c/C: creating writer at 1732159525042 (+428 ms)Flushing 6b132942b031ffcc7f2f51c4c9e8428c/C: appending metadata at 1732159525053 (+11 ms)Flushing 6b132942b031ffcc7f2f51c4c9e8428c/C: closing flushed file at 1732159525053Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24e2519e: reopening flushed file at 1732159525494 (+441 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5904e361: reopening flushed file at 1732159525514 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2952ce13: reopening flushed file at 1732159525523 (+9 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6b132942b031ffcc7f2f51c4c9e8428c in 1352ms, sequenceid=586, compaction requested=true at 1732159525533 (+10 ms)Writing region close event to WAL at 1732159525769 (+236 ms)Running coprocessor post-close hooks at 1732159525774 (+5 ms)Closed at 1732159525774 2024-11-21T03:25:25,785 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed 6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:25,788 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=6b132942b031ffcc7f2f51c4c9e8428c, regionState=CLOSED 2024-11-21T03:25:25,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:25:25,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-21T03:25:25,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure 6b132942b031ffcc7f2f51c4c9e8428c, server=b69e2a1ee061,45649,1732159441548 in 2.3830 sec 2024-11-21T03:25:25,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-21T03:25:25,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6b132942b031ffcc7f2f51c4c9e8428c, UNASSIGN in 2.3890 sec 2024-11-21T03:25:25,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-21T03:25:25,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3950 sec 2024-11-21T03:25:25,801 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159525801"}]},"ts":"1732159525801"} 2024-11-21T03:25:25,803 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T03:25:25,804 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T03:25:25,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4120 sec 2024-11-21T03:25:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:25:27,558 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:27,559 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T03:25:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:27,561 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T03:25:27,564 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:27,576 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:27,587 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/recovered.edits] 2024-11-21T03:25:27,599 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/48af540588aa41b58d1091b2b0a257ae to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/48af540588aa41b58d1091b2b0a257ae 2024-11-21T03:25:27,600 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/9f3cf08671e44d7c8d99d7651df0e44a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/9f3cf08671e44d7c8d99d7651df0e44a 2024-11-21T03:25:27,601 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/b0e77e2362724bb4b3542999e67e3421 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/A/b0e77e2362724bb4b3542999e67e3421 2024-11-21T03:25:27,603 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/465afe1cf6714b339f4f884148527d7f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/465afe1cf6714b339f4f884148527d7f 2024-11-21T03:25:27,604 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/abfbf3ed994248c5bd2cdc502d1c596f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/abfbf3ed994248c5bd2cdc502d1c596f 2024-11-21T03:25:27,605 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/eda27034fb5b402ebcbfe972edc15b1b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/B/eda27034fb5b402ebcbfe972edc15b1b 2024-11-21T03:25:27,606 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/35ae3d21862c433384af79cd2bcf57db to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/35ae3d21862c433384af79cd2bcf57db 2024-11-21T03:25:27,607 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/8a029f42ba5549f684fbfc040141ef8a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/8a029f42ba5549f684fbfc040141ef8a 2024-11-21T03:25:27,609 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d2f24b4776d04a3fb3dedb8fea6491fd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/C/d2f24b4776d04a3fb3dedb8fea6491fd 2024-11-21T03:25:27,616 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/recovered.edits/589.seqid to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c/recovered.edits/589.seqid 2024-11-21T03:25:27,616 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/6b132942b031ffcc7f2f51c4c9e8428c 2024-11-21T03:25:27,616 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T03:25:27,619 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:27,623 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T03:25:27,625 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T03:25:27,626 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:27,626 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T03:25:27,627 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732159527626"}]},"ts":"9223372036854775807"} 2024-11-21T03:25:27,629 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T03:25:27,629 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6b132942b031ffcc7f2f51c4c9e8428c, NAME => 'TestAcidGuarantees,,1732159501609.6b132942b031ffcc7f2f51c4c9e8428c.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T03:25:27,629 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T03:25:27,629 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732159527629"}]},"ts":"9223372036854775807"} 2024-11-21T03:25:27,631 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T03:25:27,632 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:27,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-21T03:25:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T03:25:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T03:25:27,670 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:27,679 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_412655455_22 at /127.0.0.1:51800 [Waiting for operation #59] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_412655455_22 at /127.0.0.1:33346 [Waiting for operation #382] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1294884874_22 at /127.0.0.1:48330 [Waiting for operation #319] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=981 (was 997), ProcessCount=11 (was 11), AvailableMemoryMB=2236 (was 3575) 2024-11-21T03:25:27,688 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=981, ProcessCount=11, AvailableMemoryMB=2236 2024-11-21T03:25:27,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:25:27,690 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:25:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:27,692 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:25:27,692 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:27,692 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-21T03:25:27,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:25:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-21T03:25:27,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742194_1370 (size=959) 2024-11-21T03:25:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-21T03:25:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-21T03:25:28,100 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1 2024-11-21T03:25:28,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742195_1371 (size=53) 2024-11-21T03:25:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-21T03:25:28,507 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:25:28,507 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cc2795dfb3064428041463dd8455a05e, disabling compactions & flushes 2024-11-21T03:25:28,507 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:28,507 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:28,507 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. after waiting 0 ms 2024-11-21T03:25:28,507 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:28,507 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:28,507 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cc2795dfb3064428041463dd8455a05e: Waiting for close lock at 1732159528507Disabling compacts and flushes for region at 1732159528507Disabling writes for close at 1732159528507Writing region close event to WAL at 1732159528507Closed at 1732159528507 2024-11-21T03:25:28,508 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:25:28,509 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732159528509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732159528509"}]},"ts":"1732159528509"} 2024-11-21T03:25:28,511 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:25:28,513 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:25:28,513 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159528513"}]},"ts":"1732159528513"} 2024-11-21T03:25:28,515 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:25:28,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, ASSIGN}] 2024-11-21T03:25:28,517 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, ASSIGN 2024-11-21T03:25:28,518 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, ASSIGN; state=OFFLINE, location=b69e2a1ee061,45649,1732159441548; forceNewPlan=false, retain=false 2024-11-21T03:25:28,669 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=cc2795dfb3064428041463dd8455a05e, regionState=OPENING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:28,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, ASSIGN because future has completed 2024-11-21T03:25:28,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:25:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-21T03:25:28,843 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:28,843 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:25:28,844 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,844 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:25:28,844 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,844 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,846 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,856 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc2795dfb3064428041463dd8455a05e columnFamilyName A 2024-11-21T03:25:28,856 DEBUG [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:28,859 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(327): Store=cc2795dfb3064428041463dd8455a05e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:25:28,860 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,865 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc2795dfb3064428041463dd8455a05e columnFamilyName B 2024-11-21T03:25:28,865 DEBUG [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:28,868 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(327): Store=cc2795dfb3064428041463dd8455a05e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:25:28,869 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,870 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc2795dfb3064428041463dd8455a05e columnFamilyName C 2024-11-21T03:25:28,871 DEBUG [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:28,872 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(327): Store=cc2795dfb3064428041463dd8455a05e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:25:28,873 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,873 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,874 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,875 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,875 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,875 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:25:28,876 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,881 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:25:28,881 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened cc2795dfb3064428041463dd8455a05e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64723468, jitterRate=-0.035545170307159424}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:25:28,881 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:28,882 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for cc2795dfb3064428041463dd8455a05e: Running coprocessor pre-open hook at 1732159528844Writing region info on filesystem at 1732159528844Initializing all the Stores at 1732159528845 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159528845Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159528846 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159528846Cleaning up temporary data from old regions at 1732159528875 (+29 ms)Running coprocessor post-open hooks at 1732159528881 (+6 ms)Region opened successfully at 1732159528882 (+1 ms) 2024-11-21T03:25:28,883 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., pid=81, masterSystemTime=1732159528839 2024-11-21T03:25:28,885 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:28,885 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:28,886 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=cc2795dfb3064428041463dd8455a05e, regionState=OPEN, openSeqNum=2, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:28,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:25:28,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-21T03:25:28,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 in 210 msec 2024-11-21T03:25:28,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-21T03:25:28,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, ASSIGN in 376 msec 2024-11-21T03:25:28,895 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:25:28,896 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159528895"}]},"ts":"1732159528895"} 2024-11-21T03:25:28,898 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:25:28,899 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:25:28,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2090 sec 2024-11-21T03:25:29,317 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T03:25:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-21T03:25:29,838 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:29,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:29,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:29,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:29,840 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:29,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402acd7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:29,842 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:29,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f3e70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:29,843 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:29,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:29,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:29,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:29,845 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:25:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b69e2a1ee061,42023,1732159440464 2024-11-21T03:25:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f576c9 2024-11-21T03:25:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:25:29,847 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:25:29,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:25:29,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:25:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:29,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742196_1372 (size=995) 2024-11-21T03:25:30,262 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T03:25:30,262 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T03:25:30,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:25:30,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, REOPEN/MOVE}] 2024-11-21T03:25:30,282 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, REOPEN/MOVE 2024-11-21T03:25:30,285 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=cc2795dfb3064428041463dd8455a05e, regionState=CLOSING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:30,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, REOPEN/MOVE because future has completed 2024-11-21T03:25:30,293 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:25:30,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:25:30,451 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,451 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:25:30,451 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing cc2795dfb3064428041463dd8455a05e, disabling compactions & flushes 2024-11-21T03:25:30,451 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:30,451 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:30,451 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. after waiting 0 ms 2024-11-21T03:25:30,451 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:30,455 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T03:25:30,455 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:30,455 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for cc2795dfb3064428041463dd8455a05e: Waiting for close lock at 1732159530451Running coprocessor pre-close hooks at 1732159530451Disabling compacts and flushes for region at 1732159530451Disabling writes for close at 1732159530451Writing region close event to WAL at 1732159530452 (+1 ms)Running coprocessor post-close hooks at 1732159530455 (+3 ms)Closed at 1732159530455 2024-11-21T03:25:30,455 WARN [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: cc2795dfb3064428041463dd8455a05e to self. 2024-11-21T03:25:30,457 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,458 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=cc2795dfb3064428041463dd8455a05e, regionState=CLOSED 2024-11-21T03:25:30,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:25:30,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-21T03:25:30,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 in 168 msec 2024-11-21T03:25:30,464 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, REOPEN/MOVE; state=CLOSED, location=b69e2a1ee061,45649,1732159441548; forceNewPlan=false, retain=true 2024-11-21T03:25:30,614 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=cc2795dfb3064428041463dd8455a05e, regionState=OPENING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:30,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, REOPEN/MOVE because future has completed 2024-11-21T03:25:30,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:25:30,771 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:30,771 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:25:30,771 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,771 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:25:30,771 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,771 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,773 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,773 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc2795dfb3064428041463dd8455a05e columnFamilyName A 2024-11-21T03:25:30,774 DEBUG [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:30,775 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(327): Store=cc2795dfb3064428041463dd8455a05e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:25:30,775 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,776 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc2795dfb3064428041463dd8455a05e columnFamilyName B 2024-11-21T03:25:30,776 DEBUG [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:30,777 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(327): Store=cc2795dfb3064428041463dd8455a05e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:25:30,777 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,777 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc2795dfb3064428041463dd8455a05e columnFamilyName C 2024-11-21T03:25:30,777 DEBUG [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:30,779 INFO [StoreOpener-cc2795dfb3064428041463dd8455a05e-1 {}] regionserver.HStore(327): Store=cc2795dfb3064428041463dd8455a05e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:25:30,779 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,779 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,780 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,781 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,781 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,781 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:25:30,782 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,783 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened cc2795dfb3064428041463dd8455a05e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72407862, jitterRate=0.07896122336387634}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:25:30,783 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,784 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for cc2795dfb3064428041463dd8455a05e: Running coprocessor pre-open hook at 1732159530771Writing region info on filesystem at 1732159530771Initializing all the Stores at 1732159530772 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159530772Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159530772Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159530772Cleaning up temporary data from old regions at 1732159530781 (+9 ms)Running coprocessor post-open hooks at 1732159530783 (+2 ms)Region opened successfully at 1732159530783 2024-11-21T03:25:30,784 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., pid=86, masterSystemTime=1732159530768 2024-11-21T03:25:30,786 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:30,786 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:30,787 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=cc2795dfb3064428041463dd8455a05e, regionState=OPEN, openSeqNum=5, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:30,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:25:30,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-21T03:25:30,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 in 173 msec 2024-11-21T03:25:30,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-21T03:25:30,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, REOPEN/MOVE in 510 msec 2024-11-21T03:25:30,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-21T03:25:30,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-11-21T03:25:30,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 950 msec 2024-11-21T03:25:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T03:25:30,802 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:30,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:30,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:30,804 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:30,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0f9435, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:30,806 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739fe7e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:30,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:30,809 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e9246e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:30,810 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc557ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:30,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:30,812 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32bdb401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:30,814 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8400d5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:30,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:30,816 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13bf07be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:30,817 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:30,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d6a5792, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:30,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:30,819 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d017ab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:30,821 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe0b6c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:30,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:30,822 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2927f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:30,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:30,824 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:30,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dfad35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:30,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:30,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24aa62e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:30,827 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4266b848, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:30,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:30,829 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18263035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:30,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:30,831 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7344faa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:30,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:30,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:30,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:30,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:30,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@151f301f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:30,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:30,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:30,836 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283397e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:25:30,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:25:30,842 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:25:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:25:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:25:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@afb8b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:25:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:25:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:30,845 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:25:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3969c387, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:25:30,848 DEBUG [Thread-1677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:30,849 DEBUG [Thread-1681 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:30,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:30,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:30,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:25:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:25:30,857 DEBUG [Thread-1696 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:30,858 DEBUG [Thread-1690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:30,860 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:30,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:25:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:25:30,869 DEBUG [Thread-1694 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:30,871 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:30,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:25:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:30,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:30,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,873 DEBUG [Thread-1685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:30,874 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:30,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:25:30,889 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:25:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-21T03:25:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-21T03:25:30,892 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:30,893 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:30,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:30,923 DEBUG [Thread-1688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:30,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:25:30,930 DEBUG [Thread-1692 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:30,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:30,936 DEBUG [Thread-1679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:25:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:30,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:25:30,949 DEBUG [Thread-1683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:25:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:25:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:25:30,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:25:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:25:30,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d006147947d042e8b5b504c129f2f634_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159530865/Put/seqid=0 2024-11-21T03:25:30,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159590955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:30,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159590958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:30,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:30,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159590962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:30,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159590966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:30,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:30,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159590974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:30,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742197_1373 (size=17034) 2024-11-21T03:25:30,989 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:30,995 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d006147947d042e8b5b504c129f2f634_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d006147947d042e8b5b504c129f2f634_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:30,996 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/b00ae7a7fcfe452d91ec66159963f89a, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:30,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/b00ae7a7fcfe452d91ec66159963f89a is 175, key is test_row_0/A:col10/1732159530865/Put/seqid=0 2024-11-21T03:25:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-21T03:25:31,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742198_1374 (size=48139) 2024-11-21T03:25:31,034 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/b00ae7a7fcfe452d91ec66159963f89a 2024-11-21T03:25:31,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-21T03:25:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/1a7b524780ec4401ba3663169f74455b is 50, key is test_row_0/B:col10/1732159530865/Put/seqid=0 2024-11-21T03:25:31,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159591078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159591078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159591078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159591088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159591088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742199_1375 (size=12001) 2024-11-21T03:25:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-21T03:25:31,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-21T03:25:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159591298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159591298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159591298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,304 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:25:31,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159591307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159591307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-21T03:25:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-21T03:25:31,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-21T03:25:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/1a7b524780ec4401ba3663169f74455b 2024-11-21T03:25:31,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/1d8ca631b9fa4e1baa95f4db3614b2cc is 50, key is test_row_0/C:col10/1732159530865/Put/seqid=0 2024-11-21T03:25:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159591609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159591609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159591609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742200_1376 (size=12001) 2024-11-21T03:25:31,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/1d8ca631b9fa4e1baa95f4db3614b2cc 2024-11-21T03:25:31,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159591617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159591620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:31,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:31,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/b00ae7a7fcfe452d91ec66159963f89a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b00ae7a7fcfe452d91ec66159963f89a 2024-11-21T03:25:31,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b00ae7a7fcfe452d91ec66159963f89a, entries=250, sequenceid=15, filesize=47.0 K 2024-11-21T03:25:31,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-21T03:25:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:31,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/1a7b524780ec4401ba3663169f74455b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1a7b524780ec4401ba3663169f74455b 2024-11-21T03:25:31,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1a7b524780ec4401ba3663169f74455b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T03:25:31,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/1d8ca631b9fa4e1baa95f4db3614b2cc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/1d8ca631b9fa4e1baa95f4db3614b2cc 2024-11-21T03:25:31,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/1d8ca631b9fa4e1baa95f4db3614b2cc, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T03:25:31,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for cc2795dfb3064428041463dd8455a05e in 880ms, sequenceid=15, compaction requested=false 2024-11-21T03:25:31,751 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T03:25:31,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:31,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-21T03:25:31,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:31,830 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:25:31,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210cfecd87ac7d440587d60bd79df1b4e3_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159530965/Put/seqid=0 2024-11-21T03:25:31,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742201_1377 (size=12154) 2024-11-21T03:25:31,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:31,908 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210cfecd87ac7d440587d60bd79df1b4e3_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210cfecd87ac7d440587d60bd79df1b4e3_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/bb0bfcebbafb4802853568a8a6bdcdc6, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/bb0bfcebbafb4802853568a8a6bdcdc6 is 175, key is test_row_0/A:col10/1732159530965/Put/seqid=0 2024-11-21T03:25:31,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742202_1378 (size=30955) 2024-11-21T03:25:31,957 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/bb0bfcebbafb4802853568a8a6bdcdc6 2024-11-21T03:25:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/6da1602c98334a5896e2e30f578786fc is 50, key is test_row_0/B:col10/1732159530965/Put/seqid=0 2024-11-21T03:25:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-21T03:25:32,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742203_1379 (size=12001) 2024-11-21T03:25:32,044 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/6da1602c98334a5896e2e30f578786fc 2024-11-21T03:25:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/322ee2d5a22b400aace4ca2f34653dc6 is 50, key is test_row_0/C:col10/1732159530965/Put/seqid=0 2024-11-21T03:25:32,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742204_1380 (size=12001) 2024-11-21T03:25:32,098 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/322ee2d5a22b400aace4ca2f34653dc6 2024-11-21T03:25:32,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/bb0bfcebbafb4802853568a8a6bdcdc6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/bb0bfcebbafb4802853568a8a6bdcdc6 2024-11-21T03:25:32,121 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/bb0bfcebbafb4802853568a8a6bdcdc6, entries=150, sequenceid=40, filesize=30.2 K 2024-11-21T03:25:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/6da1602c98334a5896e2e30f578786fc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6da1602c98334a5896e2e30f578786fc 2024-11-21T03:25:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,143 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6da1602c98334a5896e2e30f578786fc, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T03:25:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/322ee2d5a22b400aace4ca2f34653dc6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/322ee2d5a22b400aace4ca2f34653dc6 2024-11-21T03:25:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,155 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/322ee2d5a22b400aace4ca2f34653dc6, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T03:25:32,156 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for cc2795dfb3064428041463dd8455a05e in 326ms, sequenceid=40, compaction requested=false 2024-11-21T03:25:32,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:32,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:32,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-21T03:25:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-21T03:25:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-21T03:25:32,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2650 sec 2024-11-21T03:25:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 1.2730 sec 2024-11-21T03:25:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:32,357 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:32,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:25:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121791873e224ff44a286122d73be539d2d_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159532355/Put/seqid=0 2024-11-21T03:25:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742206_1382 (size=24358) 2024-11-21T03:25:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159592465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159592479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159592486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159592487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159592488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159592597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159592597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159592607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159592608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159592608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159592807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,836 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:32,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159592808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159592828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159592828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,842 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121791873e224ff44a286122d73be539d2d_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121791873e224ff44a286122d73be539d2d_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,844 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/11f932732c9c473cb42cdecb468af635, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/11f932732c9c473cb42cdecb468af635 is 175, key is test_row_0/A:col10/1732159532355/Put/seqid=0 2024-11-21T03:25:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:32,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742205_1381 (size=73994) 2024-11-21T03:25:32,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159592832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:32,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-21T03:25:33,039 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:33,039 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-21T03:25:33,042 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:33,043 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:33,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T03:25:33,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159593128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:33,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T03:25:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159593148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159593148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159593148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159593168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-21T03:25:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,249 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/11f932732c9c473cb42cdecb468af635 2024-11-21T03:25:33,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/d2feda72e3c049dfafbe64deb072c6ce is 50, key is test_row_0/B:col10/1732159532355/Put/seqid=0 2024-11-21T03:25:33,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742207_1383 (size=12001) 2024-11-21T03:25:33,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/d2feda72e3c049dfafbe64deb072c6ce 2024-11-21T03:25:33,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-21T03:25:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T03:25:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/44d3629ed60f49b5bd8c9c28bf2b0ad8 is 50, key is test_row_0/C:col10/1732159532355/Put/seqid=0 2024-11-21T03:25:33,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742208_1384 (size=12001) 2024-11-21T03:25:33,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-21T03:25:33,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159593657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T03:25:33,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-21T03:25:33,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:33,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159593678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:33,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159593678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:33,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159593678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:33,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159593698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:33,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:33,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-21T03:25:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:33,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/44d3629ed60f49b5bd8c9c28bf2b0ad8 2024-11-21T03:25:33,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/11f932732c9c473cb42cdecb468af635 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/11f932732c9c473cb42cdecb468af635 2024-11-21T03:25:33,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/11f932732c9c473cb42cdecb468af635, entries=400, sequenceid=51, filesize=72.3 K 2024-11-21T03:25:33,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/d2feda72e3c049dfafbe64deb072c6ce as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/d2feda72e3c049dfafbe64deb072c6ce 2024-11-21T03:25:33,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/d2feda72e3c049dfafbe64deb072c6ce, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T03:25:33,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/44d3629ed60f49b5bd8c9c28bf2b0ad8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/44d3629ed60f49b5bd8c9c28bf2b0ad8 2024-11-21T03:25:33,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/44d3629ed60f49b5bd8c9c28bf2b0ad8, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T03:25:33,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for cc2795dfb3064428041463dd8455a05e in 1511ms, sequenceid=51, compaction requested=true 2024-11-21T03:25:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:33,869 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:33,869 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:33,870 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:33,870 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 153088 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:33,870 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/A is initiating minor compaction (all files) 2024-11-21T03:25:33,870 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/B is initiating minor compaction (all files) 2024-11-21T03:25:33,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:33,870 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/A in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,870 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/B in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,870 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b00ae7a7fcfe452d91ec66159963f89a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/bb0bfcebbafb4802853568a8a6bdcdc6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/11f932732c9c473cb42cdecb468af635] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=149.5 K 2024-11-21T03:25:33,870 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1a7b524780ec4401ba3663169f74455b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6da1602c98334a5896e2e30f578786fc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/d2feda72e3c049dfafbe64deb072c6ce] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=35.2 K 2024-11-21T03:25:33,870 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,870 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b00ae7a7fcfe452d91ec66159963f89a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/bb0bfcebbafb4802853568a8a6bdcdc6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/11f932732c9c473cb42cdecb468af635] 2024-11-21T03:25:33,872 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a7b524780ec4401ba3663169f74455b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732159530860 2024-11-21T03:25:33,872 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b00ae7a7fcfe452d91ec66159963f89a, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732159530860 2024-11-21T03:25:33,872 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da1602c98334a5896e2e30f578786fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732159530953 2024-11-21T03:25:33,872 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb0bfcebbafb4802853568a8a6bdcdc6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732159530953 2024-11-21T03:25:33,873 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11f932732c9c473cb42cdecb468af635, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159532318 2024-11-21T03:25:33,873 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d2feda72e3c049dfafbe64deb072c6ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159532339 2024-11-21T03:25:33,889 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:33,906 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#B#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:33,907 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/faeb4f98d1be4aa5a4aa41da12efcfeb is 50, key is test_row_0/B:col10/1732159532355/Put/seqid=0 2024-11-21T03:25:33,917 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121477f703bd8af4e24a8472c5e84e4c9e0_cc2795dfb3064428041463dd8455a05e store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:33,920 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121477f703bd8af4e24a8472c5e84e4c9e0_cc2795dfb3064428041463dd8455a05e, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:33,920 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121477f703bd8af4e24a8472c5e84e4c9e0_cc2795dfb3064428041463dd8455a05e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:33,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742209_1385 (size=12104) 2024-11-21T03:25:33,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742210_1386 (size=4469) 2024-11-21T03:25:33,977 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/faeb4f98d1be4aa5a4aa41da12efcfeb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/faeb4f98d1be4aa5a4aa41da12efcfeb 2024-11-21T03:25:33,977 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#A#compaction#330 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:33,978 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/1b4b54b3ba394396aa052fbb4aa74041 is 175, key is test_row_0/A:col10/1732159532355/Put/seqid=0 2024-11-21T03:25:33,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-21T03:25:33,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,980 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T03:25:33,988 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/B of cc2795dfb3064428041463dd8455a05e into faeb4f98d1be4aa5a4aa41da12efcfeb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:33,988 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:33,988 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/B, priority=13, startTime=1732159533869; duration=0sec 2024-11-21T03:25:33,989 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:33,989 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:B 2024-11-21T03:25:33,989 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:33,990 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:33,990 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/C is initiating minor compaction (all files) 2024-11-21T03:25:33,990 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/C in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:33,990 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/1d8ca631b9fa4e1baa95f4db3614b2cc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/322ee2d5a22b400aace4ca2f34653dc6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/44d3629ed60f49b5bd8c9c28bf2b0ad8] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=35.2 K 2024-11-21T03:25:33,990 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d8ca631b9fa4e1baa95f4db3614b2cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732159530860 2024-11-21T03:25:33,991 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 322ee2d5a22b400aace4ca2f34653dc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732159530953 2024-11-21T03:25:33,991 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 44d3629ed60f49b5bd8c9c28bf2b0ad8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159532339 2024-11-21T03:25:34,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219e0c3eaf03c449e6ab186b5b95b8b9bb_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159532476/Put/seqid=0 2024-11-21T03:25:34,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742211_1387 (size=31058) 2024-11-21T03:25:34,016 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#C#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:34,016 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/488efb1b10644008a9c7261739aa9d17 is 50, key is test_row_0/C:col10/1732159532355/Put/seqid=0 2024-11-21T03:25:34,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742212_1388 (size=12154) 2024-11-21T03:25:34,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:34,042 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219e0c3eaf03c449e6ab186b5b95b8b9bb_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e0c3eaf03c449e6ab186b5b95b8b9bb_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:34,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/647474e4fe174948adaa1c1cc4a56195, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:34,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/647474e4fe174948adaa1c1cc4a56195 is 175, key is test_row_0/A:col10/1732159532476/Put/seqid=0 2024-11-21T03:25:34,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742213_1389 (size=12104) 2024-11-21T03:25:34,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742214_1390 (size=30955) 2024-11-21T03:25:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T03:25:34,431 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/1b4b54b3ba394396aa052fbb4aa74041 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1b4b54b3ba394396aa052fbb4aa74041 2024-11-21T03:25:34,448 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/A of cc2795dfb3064428041463dd8455a05e into 1b4b54b3ba394396aa052fbb4aa74041(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:34,448 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:34,448 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/A, priority=13, startTime=1732159533869; duration=0sec 2024-11-21T03:25:34,448 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:34,448 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:A 2024-11-21T03:25:34,473 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/488efb1b10644008a9c7261739aa9d17 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/488efb1b10644008a9c7261739aa9d17 2024-11-21T03:25:34,480 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/C of cc2795dfb3064428041463dd8455a05e into 488efb1b10644008a9c7261739aa9d17(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:34,480 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:34,480 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/C, priority=13, startTime=1732159533869; duration=0sec 2024-11-21T03:25:34,480 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:34,481 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:C 2024-11-21T03:25:34,482 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/647474e4fe174948adaa1c1cc4a56195 2024-11-21T03:25:34,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/17285a2540d742caaaf8771f8c73be5a is 50, key is test_row_0/B:col10/1732159532476/Put/seqid=0 2024-11-21T03:25:34,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742215_1391 (size=12001) 2024-11-21T03:25:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:34,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159594719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:34,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159594728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:34,785 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_1' in cc2795dfb3064428041463dd8455a05e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2299 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:25:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:34,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159594772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:34,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159594773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:34,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159594776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:34,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159594878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159594898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159594898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:34,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159594898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:34,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:34,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:34,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:34,964 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/17285a2540d742caaaf8771f8c73be5a 2024-11-21T03:25:34,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/f2b24691ec7e4b3b84e1b8321e4eb3ec is 50, key is test_row_0/C:col10/1732159532476/Put/seqid=0 2024-11-21T03:25:34,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742216_1392 (size=12001) 2024-11-21T03:25:34,987 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/f2b24691ec7e4b3b84e1b8321e4eb3ec 2024-11-21T03:25:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/647474e4fe174948adaa1c1cc4a56195 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/647474e4fe174948adaa1c1cc4a56195 2024-11-21T03:25:35,002 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/647474e4fe174948adaa1c1cc4a56195, entries=150, sequenceid=77, filesize=30.2 K 2024-11-21T03:25:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/17285a2540d742caaaf8771f8c73be5a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/17285a2540d742caaaf8771f8c73be5a 2024-11-21T03:25:35,008 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/17285a2540d742caaaf8771f8c73be5a, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T03:25:35,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/f2b24691ec7e4b3b84e1b8321e4eb3ec as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f2b24691ec7e4b3b84e1b8321e4eb3ec 2024-11-21T03:25:35,019 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f2b24691ec7e4b3b84e1b8321e4eb3ec, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T03:25:35,021 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for cc2795dfb3064428041463dd8455a05e in 1041ms, sequenceid=77, compaction requested=false 2024-11-21T03:25:35,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:35,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:35,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-21T03:25:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-21T03:25:35,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-21T03:25:35,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9810 sec 2024-11-21T03:25:35,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.9920 sec 2024-11-21T03:25:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:35,104 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:35,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:25:35,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a2efd05e7a24487585b13db0941550cb_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159535102/Put/seqid=0 2024-11-21T03:25:35,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742217_1393 (size=14594) 2024-11-21T03:25:35,139 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:35,145 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a2efd05e7a24487585b13db0941550cb_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a2efd05e7a24487585b13db0941550cb_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:35,147 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/82d3dcfef8094165b5b0dae8f1d1086c, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:35,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/82d3dcfef8094165b5b0dae8f1d1086c is 175, key is test_row_0/A:col10/1732159535102/Put/seqid=0 2024-11-21T03:25:35,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742218_1394 (size=39549) 2024-11-21T03:25:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T03:25:35,198 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:35,199 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-21T03:25:35,202 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:35,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159595189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,204 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:35,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:35,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159595191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T03:25:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159595194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159595194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159595307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T03:25:35,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159595317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159595318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159595328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-21T03:25:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:35,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159595518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T03:25:35,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-21T03:25:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:35,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159595528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159595529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159595538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/82d3dcfef8094165b5b0dae8f1d1086c 2024-11-21T03:25:35,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/4e76ba7483ca47e396f8c4c0acffa6f4 is 50, key is test_row_0/B:col10/1732159535102/Put/seqid=0 2024-11-21T03:25:35,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742219_1395 (size=12001) 2024-11-21T03:25:35,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/4e76ba7483ca47e396f8c4c0acffa6f4 2024-11-21T03:25:35,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/fbf20d9eb5e34f40b6d905ebcee01e42 is 50, key is test_row_0/C:col10/1732159535102/Put/seqid=0 2024-11-21T03:25:35,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742220_1396 (size=12001) 2024-11-21T03:25:35,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-21T03:25:35,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:35,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:35,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:35,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T03:25:35,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159595837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-21T03:25:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:35,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:35,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159595848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159595848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:35,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159595858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:35,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-21T03:25:36,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:36,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:36,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:36,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:36,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:36,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/fbf20d9eb5e34f40b6d905ebcee01e42 2024-11-21T03:25:36,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/82d3dcfef8094165b5b0dae8f1d1086c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/82d3dcfef8094165b5b0dae8f1d1086c 2024-11-21T03:25:36,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/82d3dcfef8094165b5b0dae8f1d1086c, entries=200, sequenceid=93, filesize=38.6 K 2024-11-21T03:25:36,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/4e76ba7483ca47e396f8c4c0acffa6f4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/4e76ba7483ca47e396f8c4c0acffa6f4 2024-11-21T03:25:36,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/4e76ba7483ca47e396f8c4c0acffa6f4, entries=150, sequenceid=93, filesize=11.7 K 2024-11-21T03:25:36,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/fbf20d9eb5e34f40b6d905ebcee01e42 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/fbf20d9eb5e34f40b6d905ebcee01e42 2024-11-21T03:25:36,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/fbf20d9eb5e34f40b6d905ebcee01e42, entries=150, sequenceid=93, filesize=11.7 K 2024-11-21T03:25:36,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for cc2795dfb3064428041463dd8455a05e in 981ms, sequenceid=93, compaction requested=true 2024-11-21T03:25:36,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:36,086 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:36,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:36,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:36,091 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:36,091 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/A is initiating minor compaction (all files) 2024-11-21T03:25:36,091 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/A in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:36,091 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1b4b54b3ba394396aa052fbb4aa74041, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/647474e4fe174948adaa1c1cc4a56195, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/82d3dcfef8094165b5b0dae8f1d1086c] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=99.2 K 2024-11-21T03:25:36,091 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:36,092 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1b4b54b3ba394396aa052fbb4aa74041, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/647474e4fe174948adaa1c1cc4a56195, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/82d3dcfef8094165b5b0dae8f1d1086c] 2024-11-21T03:25:36,092 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:36,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:36,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:36,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:36,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:36,092 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b4b54b3ba394396aa052fbb4aa74041, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159532339 2024-11-21T03:25:36,093 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 647474e4fe174948adaa1c1cc4a56195, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732159532476 2024-11-21T03:25:36,093 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 82d3dcfef8094165b5b0dae8f1d1086c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732159534715 2024-11-21T03:25:36,093 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:36,093 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/B is initiating minor compaction (all files) 2024-11-21T03:25:36,093 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/B in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:36,093 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/faeb4f98d1be4aa5a4aa41da12efcfeb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/17285a2540d742caaaf8771f8c73be5a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/4e76ba7483ca47e396f8c4c0acffa6f4] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=35.3 K 2024-11-21T03:25:36,094 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting faeb4f98d1be4aa5a4aa41da12efcfeb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159532339 2024-11-21T03:25:36,094 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17285a2540d742caaaf8771f8c73be5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732159532476 2024-11-21T03:25:36,094 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e76ba7483ca47e396f8c4c0acffa6f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732159534733 2024-11-21T03:25:36,102 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:36,103 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/77fa12fa4f10431291bca65f3df36da3 is 50, key is test_row_0/B:col10/1732159535102/Put/seqid=0 2024-11-21T03:25:36,107 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:36,132 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411210f7c88901a694ae0bc07991179e57211_cc2795dfb3064428041463dd8455a05e store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:36,134 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411210f7c88901a694ae0bc07991179e57211_cc2795dfb3064428041463dd8455a05e, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:36,134 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210f7c88901a694ae0bc07991179e57211_cc2795dfb3064428041463dd8455a05e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:36,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-21T03:25:36,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:36,158 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:25:36,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742221_1397 (size=12207) 2024-11-21T03:25:36,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742222_1398 (size=4469) 2024-11-21T03:25:36,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e13615b7e6a540dfb2e09477f435b8b9_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159535192/Put/seqid=0 2024-11-21T03:25:36,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742223_1399 (size=12154) 2024-11-21T03:25:36,275 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:25:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T03:25:36,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:36,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159596401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159596416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159596419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159596420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159596530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159596532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159596540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159596550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,581 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/77fa12fa4f10431291bca65f3df36da3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/77fa12fa4f10431291bca65f3df36da3 2024-11-21T03:25:36,593 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#A#compaction#340 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:36,594 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/1100038e67d14e2f9751bf401d2cd612 is 175, key is test_row_0/A:col10/1732159535102/Put/seqid=0 2024-11-21T03:25:36,601 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/B of cc2795dfb3064428041463dd8455a05e into 77fa12fa4f10431291bca65f3df36da3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:36,601 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:36,601 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/B, priority=13, startTime=1732159536091; duration=0sec 2024-11-21T03:25:36,601 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:36,601 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:B 2024-11-21T03:25:36,601 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:36,605 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:36,606 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/C is initiating minor compaction (all files) 2024-11-21T03:25:36,606 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/C in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:36,606 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/488efb1b10644008a9c7261739aa9d17, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f2b24691ec7e4b3b84e1b8321e4eb3ec, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/fbf20d9eb5e34f40b6d905ebcee01e42] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=35.3 K 2024-11-21T03:25:36,606 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 488efb1b10644008a9c7261739aa9d17, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159532339 2024-11-21T03:25:36,607 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b24691ec7e4b3b84e1b8321e4eb3ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732159532476 2024-11-21T03:25:36,608 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbf20d9eb5e34f40b6d905ebcee01e42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732159534733 2024-11-21T03:25:36,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742224_1400 (size=31161) 2024-11-21T03:25:36,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:36,644 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:36,644 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/9a91967eecda44a0b1e3f6d1921ddc1d is 50, key is test_row_0/C:col10/1732159535102/Put/seqid=0 2024-11-21T03:25:36,656 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e13615b7e6a540dfb2e09477f435b8b9_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e13615b7e6a540dfb2e09477f435b8b9_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:36,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/354bb55e92894875b1296df228afb33c, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:36,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/354bb55e92894875b1296df228afb33c is 175, key is test_row_0/A:col10/1732159535192/Put/seqid=0 2024-11-21T03:25:36,665 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/1100038e67d14e2f9751bf401d2cd612 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1100038e67d14e2f9751bf401d2cd612 2024-11-21T03:25:36,674 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/A of cc2795dfb3064428041463dd8455a05e into 1100038e67d14e2f9751bf401d2cd612(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:36,674 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:36,674 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/A, priority=13, startTime=1732159536086; duration=0sec 2024-11-21T03:25:36,675 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:36,675 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:A 2024-11-21T03:25:36,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742225_1401 (size=12207) 2024-11-21T03:25:36,687 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/9a91967eecda44a0b1e3f6d1921ddc1d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/9a91967eecda44a0b1e3f6d1921ddc1d 2024-11-21T03:25:36,695 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/C of cc2795dfb3064428041463dd8455a05e into 9a91967eecda44a0b1e3f6d1921ddc1d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:36,695 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:36,695 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/C, priority=13, startTime=1732159536092; duration=0sec 2024-11-21T03:25:36,695 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:36,695 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:C 2024-11-21T03:25:36,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742226_1402 (size=30955) 2024-11-21T03:25:36,720 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/354bb55e92894875b1296df228afb33c 2024-11-21T03:25:36,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/337637a1f186441b8f42e7b5810c99f7 is 50, key is test_row_0/B:col10/1732159535192/Put/seqid=0 2024-11-21T03:25:36,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159596737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159596758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159596770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159596773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742227_1403 (size=12001) 2024-11-21T03:25:36,804 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/337637a1f186441b8f42e7b5810c99f7 2024-11-21T03:25:36,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/36bad9ea7b724a4ea07e0aa6beb856ce is 50, key is test_row_0/C:col10/1732159535192/Put/seqid=0 2024-11-21T03:25:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159596808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:36,832 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_1' in cc2795dfb3064428041463dd8455a05e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4346 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:25:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:36,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:36,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742228_1404 (size=12001) 2024-11-21T03:25:37,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159597058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159597078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159597088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159597109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,284 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/36bad9ea7b724a4ea07e0aa6beb856ce 2024-11-21T03:25:37,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/354bb55e92894875b1296df228afb33c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/354bb55e92894875b1296df228afb33c 2024-11-21T03:25:37,298 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/354bb55e92894875b1296df228afb33c, entries=150, sequenceid=116, filesize=30.2 K 2024-11-21T03:25:37,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/337637a1f186441b8f42e7b5810c99f7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/337637a1f186441b8f42e7b5810c99f7 2024-11-21T03:25:37,304 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/337637a1f186441b8f42e7b5810c99f7, entries=150, sequenceid=116, filesize=11.7 K 2024-11-21T03:25:37,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/36bad9ea7b724a4ea07e0aa6beb856ce as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/36bad9ea7b724a4ea07e0aa6beb856ce 2024-11-21T03:25:37,310 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/36bad9ea7b724a4ea07e0aa6beb856ce, entries=150, sequenceid=116, filesize=11.7 K 2024-11-21T03:25:37,311 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for cc2795dfb3064428041463dd8455a05e in 1153ms, sequenceid=116, compaction requested=false 2024-11-21T03:25:37,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:37,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-21T03:25:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-21T03:25:37,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-21T03:25:37,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1090 sec 2024-11-21T03:25:37,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.1170 sec 2024-11-21T03:25:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-21T03:25:37,369 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:37,369 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-21T03:25:37,372 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T03:25:37,373 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:37,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T03:25:37,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-21T03:25:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:37,528 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:25:37,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e2c26b8f753d44c885c8a3861d903848_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159536416/Put/seqid=0 2024-11-21T03:25:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:37,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:37,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742229_1405 (size=12204) 2024-11-21T03:25:37,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159597661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159597667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159597667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159597669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T03:25:37,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159597778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159597787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159597787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159597789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:37,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:37,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T03:25:38,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159597999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,008 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e2c26b8f753d44c885c8a3861d903848_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e2c26b8f753d44c885c8a3861d903848_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/7ca60bd20f1f43139b1f6876662b84b9, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/7ca60bd20f1f43139b1f6876662b84b9 is 175, key is test_row_0/A:col10/1732159536416/Put/seqid=0 2024-11-21T03:25:38,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159598009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159598009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159598013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742230_1406 (size=31005) 2024-11-21T03:25:38,047 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/7ca60bd20f1f43139b1f6876662b84b9 2024-11-21T03:25:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/b9434db9f838464a8c446c0afc348eb1 is 50, key is test_row_0/B:col10/1732159536416/Put/seqid=0 2024-11-21T03:25:38,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742231_1407 (size=12051) 2024-11-21T03:25:38,104 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/b9434db9f838464a8c446c0afc348eb1 2024-11-21T03:25:38,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/357aaf115ff94bb8a780847191efcadf is 50, key is test_row_0/C:col10/1732159536416/Put/seqid=0 2024-11-21T03:25:38,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742232_1408 (size=12051) 2024-11-21T03:25:38,117 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/357aaf115ff94bb8a780847191efcadf 2024-11-21T03:25:38,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/7ca60bd20f1f43139b1f6876662b84b9 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/7ca60bd20f1f43139b1f6876662b84b9 2024-11-21T03:25:38,127 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/7ca60bd20f1f43139b1f6876662b84b9, entries=150, sequenceid=133, filesize=30.3 K 2024-11-21T03:25:38,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/b9434db9f838464a8c446c0afc348eb1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/b9434db9f838464a8c446c0afc348eb1 2024-11-21T03:25:38,139 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/b9434db9f838464a8c446c0afc348eb1, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T03:25:38,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/357aaf115ff94bb8a780847191efcadf as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/357aaf115ff94bb8a780847191efcadf 2024-11-21T03:25:38,162 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/357aaf115ff94bb8a780847191efcadf, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T03:25:38,164 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for cc2795dfb3064428041463dd8455a05e in 636ms, sequenceid=133, compaction requested=true 2024-11-21T03:25:38,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:38,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:38,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-21T03:25:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-21T03:25:38,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-21T03:25:38,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 797 msec 2024-11-21T03:25:38,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 809 msec 2024-11-21T03:25:38,326 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:38,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:25:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:38,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ab246234f82044d895b32eb3b1600e61_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159537663/Put/seqid=0 2024-11-21T03:25:38,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742233_1409 (size=14794) 2024-11-21T03:25:38,357 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:38,361 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ab246234f82044d895b32eb3b1600e61_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ab246234f82044d895b32eb3b1600e61_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:38,364 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/cf837901b3f741f384118496e0717786, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:38,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/cf837901b3f741f384118496e0717786 is 175, key is test_row_0/A:col10/1732159537663/Put/seqid=0 2024-11-21T03:25:38,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742234_1410 (size=39749) 2024-11-21T03:25:38,379 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/cf837901b3f741f384118496e0717786 2024-11-21T03:25:38,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159598369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159598368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159598369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/96bc83d094424b49bf9e70db4648f61b is 50, key is test_row_0/B:col10/1732159537663/Put/seqid=0 2024-11-21T03:25:38,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159598384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742235_1411 (size=12151) 2024-11-21T03:25:38,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/96bc83d094424b49bf9e70db4648f61b 2024-11-21T03:25:38,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/be5f668c133145449b9419aa874fdd9a is 50, key is test_row_0/C:col10/1732159537663/Put/seqid=0 2024-11-21T03:25:38,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742236_1412 (size=12151) 2024-11-21T03:25:38,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159598487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159598487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159598498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T03:25:38,508 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:38,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-21T03:25:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T03:25:38,510 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:38,511 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:38,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:38,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159598508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T03:25:38,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-21T03:25:38,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:38,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:38,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:38,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:38,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:38,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159598698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159598698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159598717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159598727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:38,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:38,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-21T03:25:38,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:38,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:38,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:38,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:38,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:38,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T03:25:38,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/be5f668c133145449b9419aa874fdd9a 2024-11-21T03:25:38,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/cf837901b3f741f384118496e0717786 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/cf837901b3f741f384118496e0717786 2024-11-21T03:25:38,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/cf837901b3f741f384118496e0717786, entries=200, sequenceid=156, filesize=38.8 K 2024-11-21T03:25:38,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/96bc83d094424b49bf9e70db4648f61b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/96bc83d094424b49bf9e70db4648f61b 2024-11-21T03:25:38,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/96bc83d094424b49bf9e70db4648f61b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-21T03:25:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/be5f668c133145449b9419aa874fdd9a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/be5f668c133145449b9419aa874fdd9a 2024-11-21T03:25:38,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/be5f668c133145449b9419aa874fdd9a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-21T03:25:38,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for cc2795dfb3064428041463dd8455a05e in 627ms, sequenceid=156, compaction requested=true 2024-11-21T03:25:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:25:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:38,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T03:25:38,953 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:38,953 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:38,955 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132870 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:38,955 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/A is initiating minor compaction (all files) 2024-11-21T03:25:38,955 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/A in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:38,955 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1100038e67d14e2f9751bf401d2cd612, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/354bb55e92894875b1296df228afb33c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/7ca60bd20f1f43139b1f6876662b84b9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/cf837901b3f741f384118496e0717786] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=129.8 K 2024-11-21T03:25:38,955 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:38,955 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1100038e67d14e2f9751bf401d2cd612, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/354bb55e92894875b1296df228afb33c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/7ca60bd20f1f43139b1f6876662b84b9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/cf837901b3f741f384118496e0717786] 2024-11-21T03:25:38,958 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:38,958 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/C is initiating minor compaction (all files) 2024-11-21T03:25:38,958 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/C in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:38,958 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/9a91967eecda44a0b1e3f6d1921ddc1d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/36bad9ea7b724a4ea07e0aa6beb856ce, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/357aaf115ff94bb8a780847191efcadf, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/be5f668c133145449b9419aa874fdd9a] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=47.3 K 2024-11-21T03:25:38,958 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1100038e67d14e2f9751bf401d2cd612, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732159534733 2024-11-21T03:25:38,959 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a91967eecda44a0b1e3f6d1921ddc1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732159534733 2024-11-21T03:25:38,959 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 354bb55e92894875b1296df228afb33c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732159535149 2024-11-21T03:25:38,959 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 36bad9ea7b724a4ea07e0aa6beb856ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732159535149 2024-11-21T03:25:38,959 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ca60bd20f1f43139b1f6876662b84b9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159536393 2024-11-21T03:25:38,959 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 357aaf115ff94bb8a780847191efcadf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159536393 2024-11-21T03:25:38,959 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf837901b3f741f384118496e0717786, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732159537663 2024-11-21T03:25:38,959 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting be5f668c133145449b9419aa874fdd9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732159537663 2024-11-21T03:25:38,974 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#C#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:38,975 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/b09ee8e56bfc466badd11d9987ba2033 is 50, key is test_row_0/C:col10/1732159537663/Put/seqid=0 2024-11-21T03:25:38,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-21T03:25:38,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:38,980 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:25:38,984 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:39,009 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ab318382781c469f8af21e533e8ef82c_cc2795dfb3064428041463dd8455a05e store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:39,012 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ab318382781c469f8af21e533e8ef82c_cc2795dfb3064428041463dd8455a05e, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:39,012 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ab318382781c469f8af21e533e8ef82c_cc2795dfb3064428041463dd8455a05e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:39,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:39,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742237_1413 (size=12493) 2024-11-21T03:25:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112183b677c0301f4cf6955dba3d8bb9f17c_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159538350/Put/seqid=0 2024-11-21T03:25:39,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742238_1414 (size=4469) 2024-11-21T03:25:39,093 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#A#compaction#352 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:39,094 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/8c05328d9acd407ab1d95253a1882631 is 175, key is test_row_0/A:col10/1732159537663/Put/seqid=0 2024-11-21T03:25:39,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742239_1415 (size=12304) 2024-11-21T03:25:39,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742240_1416 (size=31447) 2024-11-21T03:25:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T03:25:39,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159599231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159599237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159599237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159599241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159599361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159599378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159599378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159599381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,468 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/b09ee8e56bfc466badd11d9987ba2033 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/b09ee8e56bfc466badd11d9987ba2033 2024-11-21T03:25:39,478 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc2795dfb3064428041463dd8455a05e/C of cc2795dfb3064428041463dd8455a05e into b09ee8e56bfc466badd11d9987ba2033(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:39,478 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:39,478 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/C, priority=12, startTime=1732159538953; duration=0sec 2024-11-21T03:25:39,478 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:39,478 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:C 2024-11-21T03:25:39,479 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:39,482 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:39,482 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/B is initiating minor compaction (all files) 2024-11-21T03:25:39,482 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/B in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:39,482 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/77fa12fa4f10431291bca65f3df36da3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/337637a1f186441b8f42e7b5810c99f7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/b9434db9f838464a8c446c0afc348eb1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/96bc83d094424b49bf9e70db4648f61b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=47.3 K 2024-11-21T03:25:39,483 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 77fa12fa4f10431291bca65f3df36da3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732159534733 2024-11-21T03:25:39,484 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 337637a1f186441b8f42e7b5810c99f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732159535149 2024-11-21T03:25:39,484 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting b9434db9f838464a8c446c0afc348eb1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159536393 2024-11-21T03:25:39,485 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 96bc83d094424b49bf9e70db4648f61b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732159537663 2024-11-21T03:25:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:39,531 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:39,532 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/e495d84813304f3687e42a2f93465e8c is 50, key is test_row_0/B:col10/1732159537663/Put/seqid=0 2024-11-21T03:25:39,534 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112183b677c0301f4cf6955dba3d8bb9f17c_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112183b677c0301f4cf6955dba3d8bb9f17c_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:39,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/9927be30e71f46cc9a4b4fcbde58f073, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:39,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/9927be30e71f46cc9a4b4fcbde58f073 is 175, key is test_row_0/A:col10/1732159538350/Put/seqid=0 2024-11-21T03:25:39,546 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/8c05328d9acd407ab1d95253a1882631 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/8c05328d9acd407ab1d95253a1882631 2024-11-21T03:25:39,557 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc2795dfb3064428041463dd8455a05e/A of cc2795dfb3064428041463dd8455a05e into 8c05328d9acd407ab1d95253a1882631(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:39,557 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:39,557 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/A, priority=12, startTime=1732159538953; duration=0sec 2024-11-21T03:25:39,557 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:39,557 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:A 2024-11-21T03:25:39,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742242_1418 (size=31105) 2024-11-21T03:25:39,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159599578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742241_1417 (size=12493) 2024-11-21T03:25:39,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159599598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159599599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159599599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,645 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/e495d84813304f3687e42a2f93465e8c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e495d84813304f3687e42a2f93465e8c 2024-11-21T03:25:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T03:25:39,664 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc2795dfb3064428041463dd8455a05e/B of cc2795dfb3064428041463dd8455a05e into e495d84813304f3687e42a2f93465e8c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:39,664 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:39,665 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/B, priority=12, startTime=1732159538953; duration=0sec 2024-11-21T03:25:39,665 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:39,665 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:B 2024-11-21T03:25:39,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159599900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159599928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159599928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:39,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159599929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:39,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:39,976 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/9927be30e71f46cc9a4b4fcbde58f073 2024-11-21T03:25:40,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/1f1c7770c9574687a1cdcbfa08e52254 is 50, key is test_row_0/B:col10/1732159538350/Put/seqid=0 2024-11-21T03:25:40,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742243_1419 (size=12151) 2024-11-21T03:25:40,015 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/1f1c7770c9574687a1cdcbfa08e52254 2024-11-21T03:25:40,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/aacbbf56954d4cc7aaac106ad12719f7 is 50, key is test_row_0/C:col10/1732159538350/Put/seqid=0 2024-11-21T03:25:40,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742244_1420 (size=12151) 2024-11-21T03:25:40,075 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/aacbbf56954d4cc7aaac106ad12719f7 2024-11-21T03:25:40,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/9927be30e71f46cc9a4b4fcbde58f073 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/9927be30e71f46cc9a4b4fcbde58f073 2024-11-21T03:25:40,091 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/9927be30e71f46cc9a4b4fcbde58f073, entries=150, sequenceid=169, filesize=30.4 K 2024-11-21T03:25:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/1f1c7770c9574687a1cdcbfa08e52254 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1f1c7770c9574687a1cdcbfa08e52254 2024-11-21T03:25:40,098 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1f1c7770c9574687a1cdcbfa08e52254, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T03:25:40,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/aacbbf56954d4cc7aaac106ad12719f7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/aacbbf56954d4cc7aaac106ad12719f7 2024-11-21T03:25:40,114 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/aacbbf56954d4cc7aaac106ad12719f7, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T03:25:40,115 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for cc2795dfb3064428041463dd8455a05e in 1135ms, sequenceid=169, compaction requested=false 2024-11-21T03:25:40,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:40,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:40,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-21T03:25:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-21T03:25:40,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-21T03:25:40,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6050 sec 2024-11-21T03:25:40,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.6100 sec 2024-11-21T03:25:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:40,432 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:40,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T03:25:40,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e6b68ccae4db4f589260aff16ee655e8_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159540431/Put/seqid=0 2024-11-21T03:25:40,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742245_1421 (size=14794) 2024-11-21T03:25:40,446 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:40,448 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e6b68ccae4db4f589260aff16ee655e8_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e6b68ccae4db4f589260aff16ee655e8_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:40,449 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/272c792cf45c4d7692844205e973f032, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:40,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/272c792cf45c4d7692844205e973f032 is 175, key is test_row_0/A:col10/1732159540431/Put/seqid=0 2024-11-21T03:25:40,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742246_1422 (size=39749) 2024-11-21T03:25:40,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159600458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159600461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159600462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159600463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159600578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159600578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159600578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-21T03:25:40,658 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:40,658 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-21T03:25:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:25:40,660 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:40,660 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:40,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:25:40,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159600787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159600787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159600787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T03:25:40,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:40,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:40,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:40,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:40,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45368 deadline: 1732159600877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:40,884 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_1' in cc2795dfb3064428041463dd8455a05e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8398 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:25:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:25:41,104 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/272c792cf45c4d7692844205e973f032 2024-11-21T03:25:41,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159601103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:41,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159601103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:41,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159601104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:41,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/f68e6bb15fd14644bff689b9de02cb57 is 50, key is test_row_0/B:col10/1732159540431/Put/seqid=0 2024-11-21T03:25:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:41,184 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-999727967_22 at /127.0.0.1:56038 [Receiving block BP-547451667-172.17.0.2-1732159435428:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 369ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/cluster_0b1aaf4b-5e34-6dbd-0179-21f4d40d50a7/data/data2/, blockId=1073741830, seqno=1289 2024-11-21T03:25:41,187 INFO [AsyncFSWAL-0-hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData-prefix:b69e2a1ee061,42023,1732159440464 {}] wal.AbstractFSWAL(1368): Slow sync cost: 373 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44993,DS-456f41fb-3aa2-442d-a074-7697f0ce9de3,DISK]] 2024-11-21T03:25:41,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742247_1423 (size=12151) 2024-11-21T03:25:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:25:41,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T03:25:41,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:41,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:41,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:41,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159601478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:41,488 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in cc2795dfb3064428041463dd8455a05e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:25:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:41,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T03:25:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:41,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/f68e6bb15fd14644bff689b9de02cb57 2024-11-21T03:25:41,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/5b7ddf7749cc4dd6b61ea884bf743417 is 50, key is test_row_0/C:col10/1732159540431/Put/seqid=0 2024-11-21T03:25:41,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159601617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:41,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159601618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:41,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159601628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:41,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742248_1424 (size=12151) 2024-11-21T03:25:41,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/5b7ddf7749cc4dd6b61ea884bf743417 2024-11-21T03:25:41,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T03:25:41,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:41,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:41,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:41,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/272c792cf45c4d7692844205e973f032 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/272c792cf45c4d7692844205e973f032 2024-11-21T03:25:41,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/272c792cf45c4d7692844205e973f032, entries=200, sequenceid=198, filesize=38.8 K 2024-11-21T03:25:41,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/f68e6bb15fd14644bff689b9de02cb57 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/f68e6bb15fd14644bff689b9de02cb57 2024-11-21T03:25:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/f68e6bb15fd14644bff689b9de02cb57, entries=150, sequenceid=198, filesize=11.9 K 2024-11-21T03:25:41,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/5b7ddf7749cc4dd6b61ea884bf743417 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/5b7ddf7749cc4dd6b61ea884bf743417 2024-11-21T03:25:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/5b7ddf7749cc4dd6b61ea884bf743417, entries=150, sequenceid=198, filesize=11.9 K 2024-11-21T03:25:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for cc2795dfb3064428041463dd8455a05e in 1258ms, sequenceid=198, compaction requested=true 2024-11-21T03:25:41,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:41,691 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:41,691 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,692 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:41,693 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/A is initiating minor compaction (all files) 2024-11-21T03:25:41,693 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/A in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,693 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/8c05328d9acd407ab1d95253a1882631, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/9927be30e71f46cc9a4b4fcbde58f073, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/272c792cf45c4d7692844205e973f032] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=99.9 K 2024-11-21T03:25:41,693 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,693 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/8c05328d9acd407ab1d95253a1882631, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/9927be30e71f46cc9a4b4fcbde58f073, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/272c792cf45c4d7692844205e973f032] 2024-11-21T03:25:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,693 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:41,693 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/B is initiating minor compaction (all files) 2024-11-21T03:25:41,693 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/B in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,693 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e495d84813304f3687e42a2f93465e8c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1f1c7770c9574687a1cdcbfa08e52254, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/f68e6bb15fd14644bff689b9de02cb57] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=35.9 K 2024-11-21T03:25:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,694 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c05328d9acd407ab1d95253a1882631, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732159537663 2024-11-21T03:25:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,694 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e495d84813304f3687e42a2f93465e8c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732159537663 2024-11-21T03:25:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,695 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9927be30e71f46cc9a4b4fcbde58f073, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732159538350 2024-11-21T03:25:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,695 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f1c7770c9574687a1cdcbfa08e52254, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732159538350 2024-11-21T03:25:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,696 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 272c792cf45c4d7692844205e973f032, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732159539160 2024-11-21T03:25:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,696 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting f68e6bb15fd14644bff689b9de02cb57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732159539160 2024-11-21T03:25:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,709 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,710 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,711 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/2ba3c71aab0442fe842ca41d2d05268e is 50, key is test_row_0/B:col10/1732159540431/Put/seqid=0 2024-11-21T03:25:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,711 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211158c032d1d64b988b6e70423c1d93ef_cc2795dfb3064428041463dd8455a05e store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,713 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211158c032d1d64b988b6e70423c1d93ef_cc2795dfb3064428041463dd8455a05e, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,714 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211158c032d1d64b988b6e70423c1d93ef_cc2795dfb3064428041463dd8455a05e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742249_1425 (size=12595) 2024-11-21T03:25:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742250_1426 (size=4469) 2024-11-21T03:25:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,732 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#A#compaction#361 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,733 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/3fcbb41a180d4b2c9ec93718db9b285b is 175, key is test_row_0/A:col10/1732159540431/Put/seqid=0 2024-11-21T03:25:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742251_1427 (size=31549) 2024-11-21T03:25:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,745 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/3fcbb41a180d4b2c9ec93718db9b285b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3fcbb41a180d4b2c9ec93718db9b285b 2024-11-21T03:25:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,751 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/A of cc2795dfb3064428041463dd8455a05e into 3fcbb41a180d4b2c9ec93718db9b285b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:41,751 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:41,751 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/A, priority=13, startTime=1732159541690; duration=0sec 2024-11-21T03:25:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,751 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:41,751 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:A 2024-11-21T03:25:41,751 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,752 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:41,752 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/C is initiating minor compaction (all files) 2024-11-21T03:25:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,752 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/C in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,752 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/b09ee8e56bfc466badd11d9987ba2033, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/aacbbf56954d4cc7aaac106ad12719f7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/5b7ddf7749cc4dd6b61ea884bf743417] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=35.9 K 2024-11-21T03:25:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,753 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b09ee8e56bfc466badd11d9987ba2033, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732159537663 2024-11-21T03:25:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,753 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting aacbbf56954d4cc7aaac106ad12719f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732159538350 2024-11-21T03:25:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,753 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b7ddf7749cc4dd6b61ea884bf743417, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732159539160 2024-11-21T03:25:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,761 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#C#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,762 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/477ce5b6c1284d8688c9ad13985f9387 is 50, key is test_row_0/C:col10/1732159540431/Put/seqid=0 2024-11-21T03:25:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742252_1428 (size=12595) 2024-11-21T03:25:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,777 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/477ce5b6c1284d8688c9ad13985f9387 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/477ce5b6c1284d8688c9ad13985f9387 2024-11-21T03:25:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,783 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/C of cc2795dfb3064428041463dd8455a05e into 477ce5b6c1284d8688c9ad13985f9387(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,783 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:41,783 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/C, priority=13, startTime=1732159541691; duration=0sec 2024-11-21T03:25:41,784 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:41,784 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:C 2024-11-21T03:25:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:25:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T03:25:41,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,823 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T03:25:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212bc0725040a1421fb635dd73e549fbdd_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159540460/Put/seqid=0 2024-11-21T03:25:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742253_1429 (size=9814) 2024-11-21T03:25:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,846 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212bc0725040a1421fb635dd73e549fbdd_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212bc0725040a1421fb635dd73e549fbdd_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/3263d22f2f254943a24a21144706971f, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/3263d22f2f254943a24a21144706971f is 175, key is test_row_0/A:col10/1732159540460/Put/seqid=0 2024-11-21T03:25:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742254_1430 (size=22461) 2024-11-21T03:25:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,862 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/3263d22f2f254943a24a21144706971f 2024-11-21T03:25:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/af3f636bd3e44320ba3d2b28710a10a3 is 50, key is test_row_0/B:col10/1732159540460/Put/seqid=0 2024-11-21T03:25:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742255_1431 (size=9757) 2024-11-21T03:25:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,160 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/2ba3c71aab0442fe842ca41d2d05268e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/2ba3c71aab0442fe842ca41d2d05268e 2024-11-21T03:25:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,200 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/B of cc2795dfb3064428041463dd8455a05e into 2ba3c71aab0442fe842ca41d2d05268e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,200 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:42,200 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/B, priority=13, startTime=1732159541691; duration=0sec 2024-11-21T03:25:42,200 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:42,200 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:B 2024-11-21T03:25:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,288 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/af3f636bd3e44320ba3d2b28710a10a3 2024-11-21T03:25:42,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/736b53915d52468bbfc5fb79e74e918a is 50, key is test_row_0/C:col10/1732159540460/Put/seqid=0 2024-11-21T03:25:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742256_1432 (size=9757) 2024-11-21T03:25:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,388 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/736b53915d52468bbfc5fb79e74e918a 2024-11-21T03:25:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/3263d22f2f254943a24a21144706971f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3263d22f2f254943a24a21144706971f 2024-11-21T03:25:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,413 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3263d22f2f254943a24a21144706971f, entries=100, sequenceid=210, filesize=21.9 K 2024-11-21T03:25:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/af3f636bd3e44320ba3d2b28710a10a3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/af3f636bd3e44320ba3d2b28710a10a3 2024-11-21T03:25:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,420 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/af3f636bd3e44320ba3d2b28710a10a3, entries=100, sequenceid=210, filesize=9.5 K 2024-11-21T03:25:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/736b53915d52468bbfc5fb79e74e918a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/736b53915d52468bbfc5fb79e74e918a 2024-11-21T03:25:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,445 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/736b53915d52468bbfc5fb79e74e918a, entries=100, sequenceid=210, filesize=9.5 K 2024-11-21T03:25:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,452 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for cc2795dfb3064428041463dd8455a05e in 629ms, sequenceid=210, compaction requested=false 2024-11-21T03:25:42,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:42,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:42,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-21T03:25:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-21T03:25:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-21T03:25:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7940 sec 2024-11-21T03:25:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8000 sec 2024-11-21T03:25:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:25:42,807 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,810 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:42,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:25:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:42,811 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-21T03:25:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T03:25:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,813 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,814 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:42,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112173e213f9907f4069a93489d58d53f9dc_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159542801/Put/seqid=0 2024-11-21T03:25:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T03:25:42,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742257_1433 (size=24758) 2024-11-21T03:25:42,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T03:25:42,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:42,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159603054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159603056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159603059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T03:25:43,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T03:25:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159603189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159603189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159603192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T03:25:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,356 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:43,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112173e213f9907f4069a93489d58d53f9dc_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112173e213f9907f4069a93489d58d53f9dc_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:43,385 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/25743259578e4cf28fbf75e0cb24c97c, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:43,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/25743259578e4cf28fbf75e0cb24c97c is 175, key is test_row_0/A:col10/1732159542801/Put/seqid=0 2024-11-21T03:25:43,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742258_1434 (size=74395) 2024-11-21T03:25:43,395 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/25743259578e4cf28fbf75e0cb24c97c 2024-11-21T03:25:43,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/1e4c50a783744419a565acf91cd96c61 is 50, key is test_row_0/B:col10/1732159542801/Put/seqid=0 2024-11-21T03:25:43,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T03:25:43,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159603420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159603421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159603430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T03:25:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742259_1435 (size=12151) 2024-11-21T03:25:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159603510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,528 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in cc2795dfb3064428041463dd8455a05e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:25:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T03:25:43,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159603758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159603758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159603758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:43,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T03:25:43,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:43,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/1e4c50a783744419a565acf91cd96c61 2024-11-21T03:25:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T03:25:43,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/46d9834ed5df4a5aaea9c977ab7c6641 is 50, key is test_row_0/C:col10/1732159542801/Put/seqid=0 2024-11-21T03:25:43,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T03:25:43,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:43,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:43,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:43,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742260_1436 (size=12151) 2024-11-21T03:25:44,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T03:25:44,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:44,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:44,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:44,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:44,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:44,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T03:25:44,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:44,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:44,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:44,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:44,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:44,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159604278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:44,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:44,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:44,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:44,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159604278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:44,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159604278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:44,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/46d9834ed5df4a5aaea9c977ab7c6641 2024-11-21T03:25:44,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/25743259578e4cf28fbf75e0cb24c97c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/25743259578e4cf28fbf75e0cb24c97c 2024-11-21T03:25:44,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/25743259578e4cf28fbf75e0cb24c97c, entries=400, sequenceid=222, filesize=72.7 K 2024-11-21T03:25:44,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/1e4c50a783744419a565acf91cd96c61 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1e4c50a783744419a565acf91cd96c61 2024-11-21T03:25:44,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1e4c50a783744419a565acf91cd96c61, entries=150, sequenceid=222, filesize=11.9 K 2024-11-21T03:25:44,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/46d9834ed5df4a5aaea9c977ab7c6641 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/46d9834ed5df4a5aaea9c977ab7c6641 2024-11-21T03:25:44,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/46d9834ed5df4a5aaea9c977ab7c6641, entries=150, sequenceid=222, filesize=11.9 K 2024-11-21T03:25:44,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for cc2795dfb3064428041463dd8455a05e in 1621ms, sequenceid=222, compaction requested=true 2024-11-21T03:25:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:44,432 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:44,432 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:44,433 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:44,433 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/A is initiating minor compaction (all files) 2024-11-21T03:25:44,433 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/A in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:44,433 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3fcbb41a180d4b2c9ec93718db9b285b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3263d22f2f254943a24a21144706971f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/25743259578e4cf28fbf75e0cb24c97c] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=125.4 K 2024-11-21T03:25:44,433 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:44,433 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3fcbb41a180d4b2c9ec93718db9b285b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3263d22f2f254943a24a21144706971f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/25743259578e4cf28fbf75e0cb24c97c] 2024-11-21T03:25:44,433 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fcbb41a180d4b2c9ec93718db9b285b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732159539160 2024-11-21T03:25:44,434 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:44,434 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/B is initiating minor compaction (all files) 2024-11-21T03:25:44,434 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/B in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:44,434 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/2ba3c71aab0442fe842ca41d2d05268e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/af3f636bd3e44320ba3d2b28710a10a3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1e4c50a783744419a565acf91cd96c61] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=33.7 K 2024-11-21T03:25:44,434 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3263d22f2f254943a24a21144706971f, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732159540460 2024-11-21T03:25:44,434 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ba3c71aab0442fe842ca41d2d05268e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732159539160 2024-11-21T03:25:44,434 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25743259578e4cf28fbf75e0cb24c97c, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732159542744 2024-11-21T03:25:44,435 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting af3f636bd3e44320ba3d2b28710a10a3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732159540460 2024-11-21T03:25:44,435 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e4c50a783744419a565acf91cd96c61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732159542772 2024-11-21T03:25:44,450 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:44,451 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/10e1b02ae59e427cb391c985fd7f7c67 is 50, key is test_row_0/B:col10/1732159542801/Put/seqid=0 2024-11-21T03:25:44,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T03:25:44,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:44,451 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:25:44,453 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:44,473 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411215f88ff03702f46ac9e4bb531fa3f326c_cc2795dfb3064428041463dd8455a05e store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:44,477 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411215f88ff03702f46ac9e4bb531fa3f326c_cc2795dfb3064428041463dd8455a05e, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:44,477 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215f88ff03702f46ac9e4bb531fa3f326c_cc2795dfb3064428041463dd8455a05e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:44,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742261_1437 (size=12697) 2024-11-21T03:25:44,496 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/10e1b02ae59e427cb391c985fd7f7c67 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/10e1b02ae59e427cb391c985fd7f7c67 2024-11-21T03:25:44,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215d8ad580e5b7481e987862bfdcb7a873_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159543034/Put/seqid=0 2024-11-21T03:25:44,506 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/B of cc2795dfb3064428041463dd8455a05e into 10e1b02ae59e427cb391c985fd7f7c67(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:44,506 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:44,506 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/B, priority=13, startTime=1732159544432; duration=0sec 2024-11-21T03:25:44,506 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:44,506 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:B 2024-11-21T03:25:44,506 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:44,513 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:44,513 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/C is initiating minor compaction (all files) 2024-11-21T03:25:44,513 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/C in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:44,513 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/477ce5b6c1284d8688c9ad13985f9387, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/736b53915d52468bbfc5fb79e74e918a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/46d9834ed5df4a5aaea9c977ab7c6641] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=33.7 K 2024-11-21T03:25:44,516 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 477ce5b6c1284d8688c9ad13985f9387, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732159539160 2024-11-21T03:25:44,516 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 736b53915d52468bbfc5fb79e74e918a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732159540460 2024-11-21T03:25:44,516 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 46d9834ed5df4a5aaea9c977ab7c6641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732159542772 2024-11-21T03:25:44,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742262_1438 (size=4469) 2024-11-21T03:25:44,567 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#A#compaction#370 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:44,568 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/53b6c1f180d64ef59e0410167b00815e is 175, key is test_row_0/A:col10/1732159542801/Put/seqid=0 2024-11-21T03:25:44,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742263_1439 (size=12304) 2024-11-21T03:25:44,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:44,584 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215d8ad580e5b7481e987862bfdcb7a873_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215d8ad580e5b7481e987862bfdcb7a873_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:44,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/2e85974472c54e4f82696092431a4711, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:44,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/2e85974472c54e4f82696092431a4711 is 175, key is test_row_0/A:col10/1732159543034/Put/seqid=0 2024-11-21T03:25:44,590 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:44,591 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/7423ce0852784335a0ac2503da408925 is 50, key is test_row_0/C:col10/1732159542801/Put/seqid=0 2024-11-21T03:25:44,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742266_1442 (size=12697) 2024-11-21T03:25:44,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742264_1440 (size=31651) 2024-11-21T03:25:44,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742265_1441 (size=31105) 2024-11-21T03:25:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T03:25:45,070 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/7423ce0852784335a0ac2503da408925 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7423ce0852784335a0ac2503da408925 2024-11-21T03:25:45,077 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/C of cc2795dfb3064428041463dd8455a05e into 7423ce0852784335a0ac2503da408925(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:45,077 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:45,077 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/C, priority=13, startTime=1732159544432; duration=0sec 2024-11-21T03:25:45,077 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:45,077 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:C 2024-11-21T03:25:45,078 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/2e85974472c54e4f82696092431a4711 2024-11-21T03:25:45,084 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/53b6c1f180d64ef59e0410167b00815e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/53b6c1f180d64ef59e0410167b00815e 2024-11-21T03:25:45,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/6675e4547dd6467494a3bbfd894ed30f is 50, key is test_row_0/B:col10/1732159543034/Put/seqid=0 2024-11-21T03:25:45,097 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/A of cc2795dfb3064428041463dd8455a05e into 53b6c1f180d64ef59e0410167b00815e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:45,097 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:45,097 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/A, priority=13, startTime=1732159544431; duration=0sec 2024-11-21T03:25:45,097 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:45,097 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:A 2024-11-21T03:25:45,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742267_1443 (size=12151) 2024-11-21T03:25:45,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:45,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:45,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159605336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159605339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159605340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159605448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159605459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159605459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,552 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/6675e4547dd6467494a3bbfd894ed30f 2024-11-21T03:25:45,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/7986264473e9497fbd7a786885fde979 is 50, key is test_row_0/C:col10/1732159543034/Put/seqid=0 2024-11-21T03:25:45,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742268_1444 (size=12151) 2024-11-21T03:25:45,633 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/7986264473e9497fbd7a786885fde979 2024-11-21T03:25:45,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/2e85974472c54e4f82696092431a4711 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/2e85974472c54e4f82696092431a4711 2024-11-21T03:25:45,655 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/2e85974472c54e4f82696092431a4711, entries=150, sequenceid=247, filesize=30.4 K 2024-11-21T03:25:45,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/6675e4547dd6467494a3bbfd894ed30f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6675e4547dd6467494a3bbfd894ed30f 2024-11-21T03:25:45,664 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6675e4547dd6467494a3bbfd894ed30f, entries=150, sequenceid=247, filesize=11.9 K 2024-11-21T03:25:45,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159605658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/7986264473e9497fbd7a786885fde979 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7986264473e9497fbd7a786885fde979 2024-11-21T03:25:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,673 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7986264473e9497fbd7a786885fde979, entries=150, sequenceid=247, filesize=11.9 K 2024-11-21T03:25:45,676 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for cc2795dfb3064428041463dd8455a05e in 1222ms, sequenceid=247, compaction requested=false 2024-11-21T03:25:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-21T03:25:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-21T03:25:45,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-21T03:25:45,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8640 sec 2024-11-21T03:25:45,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.8700 sec 2024-11-21T03:25:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:45,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:45,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:25:45,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112188a3d7fba35345aea6ff6bd246f5b0ba_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159545683/Put/seqid=0 2024-11-21T03:25:45,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742269_1445 (size=12454) 2024-11-21T03:25:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159605858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159605860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159605979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159605988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159605988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:46,168 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:46,180 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112188a3d7fba35345aea6ff6bd246f5b0ba_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112188a3d7fba35345aea6ff6bd246f5b0ba_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:46,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/5e9284746f4843fe997821143fa2f5f8, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:46,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/5e9284746f4843fe997821143fa2f5f8 is 175, key is test_row_0/A:col10/1732159545683/Put/seqid=0 2024-11-21T03:25:46,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159606200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:46,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159606207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:46,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742270_1446 (size=31255) 2024-11-21T03:25:46,225 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/5e9284746f4843fe997821143fa2f5f8 2024-11-21T03:25:46,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/24d5f13da16f4154b2e105f15d3f985b is 50, key is test_row_0/B:col10/1732159545683/Put/seqid=0 2024-11-21T03:25:46,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742271_1447 (size=12301) 2024-11-21T03:25:46,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/24d5f13da16f4154b2e105f15d3f985b 2024-11-21T03:25:46,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/7813694448ae414cb5ecf88a892c2667 is 50, key is test_row_0/C:col10/1732159545683/Put/seqid=0 2024-11-21T03:25:46,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742272_1448 (size=12301) 2024-11-21T03:25:46,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/7813694448ae414cb5ecf88a892c2667 2024-11-21T03:25:46,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159606498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:46,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/5e9284746f4843fe997821143fa2f5f8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5e9284746f4843fe997821143fa2f5f8 2024-11-21T03:25:46,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159606517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:46,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159606527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:46,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5e9284746f4843fe997821143fa2f5f8, entries=150, sequenceid=263, filesize=30.5 K 2024-11-21T03:25:46,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/24d5f13da16f4154b2e105f15d3f985b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/24d5f13da16f4154b2e105f15d3f985b 2024-11-21T03:25:46,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/24d5f13da16f4154b2e105f15d3f985b, entries=150, sequenceid=263, filesize=12.0 K 2024-11-21T03:25:46,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/7813694448ae414cb5ecf88a892c2667 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7813694448ae414cb5ecf88a892c2667 2024-11-21T03:25:46,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7813694448ae414cb5ecf88a892c2667, entries=150, sequenceid=263, filesize=12.0 K 2024-11-21T03:25:46,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for cc2795dfb3064428041463dd8455a05e in 887ms, sequenceid=263, compaction requested=true 2024-11-21T03:25:46,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:46,571 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:46,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:46,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:46,575 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:46,575 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:46,575 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/A is initiating minor compaction (all files) 2024-11-21T03:25:46,575 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/A in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:46,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:46,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:46,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:46,575 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/53b6c1f180d64ef59e0410167b00815e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/2e85974472c54e4f82696092431a4711, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5e9284746f4843fe997821143fa2f5f8] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=91.8 K 2024-11-21T03:25:46,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:46,576 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:46,576 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/53b6c1f180d64ef59e0410167b00815e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/2e85974472c54e4f82696092431a4711, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5e9284746f4843fe997821143fa2f5f8] 2024-11-21T03:25:46,581 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:46,581 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/B is initiating minor compaction (all files) 2024-11-21T03:25:46,581 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/B in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:46,581 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/10e1b02ae59e427cb391c985fd7f7c67, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6675e4547dd6467494a3bbfd894ed30f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/24d5f13da16f4154b2e105f15d3f985b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=36.3 K 2024-11-21T03:25:46,582 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53b6c1f180d64ef59e0410167b00815e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732159542772 2024-11-21T03:25:46,583 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 10e1b02ae59e427cb391c985fd7f7c67, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732159542772 2024-11-21T03:25:46,584 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e85974472c54e4f82696092431a4711, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732159543032 2024-11-21T03:25:46,584 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6675e4547dd6467494a3bbfd894ed30f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732159543032 2024-11-21T03:25:46,585 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 24d5f13da16f4154b2e105f15d3f985b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159545336 2024-11-21T03:25:46,586 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e9284746f4843fe997821143fa2f5f8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159545336 2024-11-21T03:25:46,602 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:46,603 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/a8cc6300c49c474ca2e932de766f5126 is 50, key is test_row_0/B:col10/1732159545683/Put/seqid=0 2024-11-21T03:25:46,614 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:46,622 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121050bebbf956247fd8096930d5c2ba584_cc2795dfb3064428041463dd8455a05e store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:46,623 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121050bebbf956247fd8096930d5c2ba584_cc2795dfb3064428041463dd8455a05e, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:46,623 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121050bebbf956247fd8096930d5c2ba584_cc2795dfb3064428041463dd8455a05e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:46,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742273_1449 (size=12949) 2024-11-21T03:25:46,674 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/a8cc6300c49c474ca2e932de766f5126 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/a8cc6300c49c474ca2e932de766f5126 2024-11-21T03:25:46,682 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/B of cc2795dfb3064428041463dd8455a05e into a8cc6300c49c474ca2e932de766f5126(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:46,682 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:46,682 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/B, priority=13, startTime=1732159546575; duration=0sec 2024-11-21T03:25:46,682 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:46,682 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:B 2024-11-21T03:25:46,682 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:25:46,683 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:25:46,683 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/C is initiating minor compaction (all files) 2024-11-21T03:25:46,683 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/C in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:46,683 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7423ce0852784335a0ac2503da408925, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7986264473e9497fbd7a786885fde979, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7813694448ae414cb5ecf88a892c2667] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=36.3 K 2024-11-21T03:25:46,684 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7423ce0852784335a0ac2503da408925, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732159542772 2024-11-21T03:25:46,684 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7986264473e9497fbd7a786885fde979, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732159543032 2024-11-21T03:25:46,684 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7813694448ae414cb5ecf88a892c2667, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159545336 2024-11-21T03:25:46,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742274_1450 (size=4469) 2024-11-21T03:25:46,706 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:46,707 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/face34f18ffd4aafa491dde5063ad6d0 is 50, key is test_row_0/C:col10/1732159545683/Put/seqid=0 2024-11-21T03:25:46,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742275_1451 (size=12949) 2024-11-21T03:25:46,748 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/face34f18ffd4aafa491dde5063ad6d0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/face34f18ffd4aafa491dde5063ad6d0 2024-11-21T03:25:46,761 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/C of cc2795dfb3064428041463dd8455a05e into face34f18ffd4aafa491dde5063ad6d0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:46,761 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:46,761 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/C, priority=13, startTime=1732159546575; duration=0sec 2024-11-21T03:25:46,761 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:46,761 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:C 2024-11-21T03:25:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T03:25:47,001 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:47,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:25:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-21T03:25:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T03:25:47,003 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:25:47,004 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:25:47,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:25:47,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:47,043 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:47,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:25:47,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121989abf283452416389a42f2c9e06df25_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159545844/Put/seqid=0 2024-11-21T03:25:47,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742276_1452 (size=14994) 2024-11-21T03:25:47,066 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:47,073 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121989abf283452416389a42f2c9e06df25_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121989abf283452416389a42f2c9e06df25_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:47,074 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/18559659865741b9994cb182d5d1fee5, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:47,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/18559659865741b9994cb182d5d1fee5 is 175, key is test_row_0/A:col10/1732159545844/Put/seqid=0 2024-11-21T03:25:47,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742277_1453 (size=39949) 2024-11-21T03:25:47,095 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#A#compaction#379 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:47,096 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/61a03151f7d14803bb769fa82ddc54ff is 175, key is test_row_0/A:col10/1732159545683/Put/seqid=0 2024-11-21T03:25:47,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742278_1454 (size=31903) 2024-11-21T03:25:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T03:25:47,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159607100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:47,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159607099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:47,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T03:25:47,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:47,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159607218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:47,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159607228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T03:25:47,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T03:25:47,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:47,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159607438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:47,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159607447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:47,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T03:25:47,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:47,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,487 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/18559659865741b9994cb182d5d1fee5 2024-11-21T03:25:47,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/6147915c572c4289aaa00800e11663e2 is 50, key is test_row_0/B:col10/1732159545844/Put/seqid=0 2024-11-21T03:25:47,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742279_1455 (size=12301) 2024-11-21T03:25:47,512 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/61a03151f7d14803bb769fa82ddc54ff as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/61a03151f7d14803bb769fa82ddc54ff 2024-11-21T03:25:47,519 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc2795dfb3064428041463dd8455a05e/A of cc2795dfb3064428041463dd8455a05e into 61a03151f7d14803bb769fa82ddc54ff(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:47,519 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:47,519 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/A, priority=13, startTime=1732159546570; duration=0sec 2024-11-21T03:25:47,519 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:47,519 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:A 2024-11-21T03:25:47,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159607508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:47,522 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in cc2795dfb3064428041463dd8455a05e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:25:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45444 deadline: 1732159607548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:47,557 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in cc2795dfb3064428041463dd8455a05e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8320 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:25:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:47,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T03:25:47,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T03:25:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159607748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:47,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159607760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:47,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:47,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:47,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T03:25:47,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:47,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/6147915c572c4289aaa00800e11663e2 2024-11-21T03:25:47,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T03:25:47,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:47,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:47,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:47,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/f406b87bdb0240329f13122974fd473b is 50, key is test_row_0/C:col10/1732159545844/Put/seqid=0 2024-11-21T03:25:48,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742280_1456 (size=12301) 2024-11-21T03:25:48,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T03:25:48,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:48,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:48,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:48,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:48,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T03:25:48,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T03:25:48,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:48,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:48,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:48,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:48,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:48,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159608268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:48,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159608287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:48,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T03:25:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:48,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:48,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:25:48,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/f406b87bdb0240329f13122974fd473b 2024-11-21T03:25:48,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/18559659865741b9994cb182d5d1fee5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/18559659865741b9994cb182d5d1fee5 2024-11-21T03:25:48,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/18559659865741b9994cb182d5d1fee5, entries=200, sequenceid=289, filesize=39.0 K 2024-11-21T03:25:48,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/6147915c572c4289aaa00800e11663e2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6147915c572c4289aaa00800e11663e2 2024-11-21T03:25:48,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6147915c572c4289aaa00800e11663e2, entries=150, sequenceid=289, filesize=12.0 K 2024-11-21T03:25:48,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/f406b87bdb0240329f13122974fd473b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f406b87bdb0240329f13122974fd473b 2024-11-21T03:25:48,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f406b87bdb0240329f13122974fd473b, entries=150, sequenceid=289, filesize=12.0 K 2024-11-21T03:25:48,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for cc2795dfb3064428041463dd8455a05e in 1408ms, sequenceid=289, compaction requested=false 2024-11-21T03:25:48,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:48,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T03:25:48,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:48,559 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:25:48,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112182d05c821f2c48b3b02d050edf275152_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159547097/Put/seqid=0 2024-11-21T03:25:48,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742281_1457 (size=12454) 2024-11-21T03:25:48,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:48,597 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112182d05c821f2c48b3b02d050edf275152_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182d05c821f2c48b3b02d050edf275152_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/fbab863df576466590f6dfb08b474a2a, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/fbab863df576466590f6dfb08b474a2a is 175, key is test_row_0/A:col10/1732159547097/Put/seqid=0 2024-11-21T03:25:48,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742282_1458 (size=31255) 2024-11-21T03:25:48,649 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/fbab863df576466590f6dfb08b474a2a 2024-11-21T03:25:48,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/edbaaf7dc4d34d978e7545ee02c0a4d9 is 50, key is test_row_0/B:col10/1732159547097/Put/seqid=0 2024-11-21T03:25:48,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742283_1459 (size=12301) 2024-11-21T03:25:49,108 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/edbaaf7dc4d34d978e7545ee02c0a4d9 2024-11-21T03:25:49,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/4356ae29adef4b5695cadf1738f889fb is 50, key is test_row_0/C:col10/1732159547097/Put/seqid=0 2024-11-21T03:25:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T03:25:49,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742284_1460 (size=12301) 2024-11-21T03:25:49,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. as already flushing 2024-11-21T03:25:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:49,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159609409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:49,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159609420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:49,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159609527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:49,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45458 deadline: 1732159609538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159609538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:49,548 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in cc2795dfb3064428041463dd8455a05e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:25:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:49,568 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/4356ae29adef4b5695cadf1738f889fb 2024-11-21T03:25:49,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/fbab863df576466590f6dfb08b474a2a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/fbab863df576466590f6dfb08b474a2a 2024-11-21T03:25:49,583 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/fbab863df576466590f6dfb08b474a2a, entries=150, sequenceid=302, filesize=30.5 K 2024-11-21T03:25:49,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/edbaaf7dc4d34d978e7545ee02c0a4d9 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/edbaaf7dc4d34d978e7545ee02c0a4d9 2024-11-21T03:25:49,592 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/edbaaf7dc4d34d978e7545ee02c0a4d9, entries=150, sequenceid=302, filesize=12.0 K 2024-11-21T03:25:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/4356ae29adef4b5695cadf1738f889fb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/4356ae29adef4b5695cadf1738f889fb 2024-11-21T03:25:49,600 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/4356ae29adef4b5695cadf1738f889fb, entries=150, sequenceid=302, filesize=12.0 K 2024-11-21T03:25:49,602 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for cc2795dfb3064428041463dd8455a05e in 1043ms, sequenceid=302, compaction requested=true 2024-11-21T03:25:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-21T03:25:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-21T03:25:49,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-21T03:25:49,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6000 sec 2024-11-21T03:25:49,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.6050 sec 2024-11-21T03:25:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:49,746 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:49,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:25:49,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112180e16322f3e8473fb8fd8d0bb7ffc072_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159549415/Put/seqid=0 2024-11-21T03:25:49,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742285_1461 (size=14994) 2024-11-21T03:25:49,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159609799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:49,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159609812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:49,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159609918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:49,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159609928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:49,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:50,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159610138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159610147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:50,175 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:50,178 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112180e16322f3e8473fb8fd8d0bb7ffc072_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112180e16322f3e8473fb8fd8d0bb7ffc072_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:50,179 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/1814ff7adcf44af388b06d3e386f8e18, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:50,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/1814ff7adcf44af388b06d3e386f8e18 is 175, key is test_row_0/A:col10/1732159549415/Put/seqid=0 2024-11-21T03:25:50,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742286_1462 (size=39949) 2024-11-21T03:25:50,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/1814ff7adcf44af388b06d3e386f8e18 2024-11-21T03:25:50,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/e2dc396c1d874723a0f9947d8267859f is 50, key is test_row_0/B:col10/1732159549415/Put/seqid=0 2024-11-21T03:25:50,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742287_1463 (size=12301) 2024-11-21T03:25:50,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45422 deadline: 1732159610449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:50,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:25:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45352 deadline: 1732159610469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:25:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:25:50,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/e2dc396c1d874723a0f9947d8267859f 2024-11-21T03:25:50,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/771a15626db24f9ea6d1cff106502977 is 50, key is test_row_0/C:col10/1732159549415/Put/seqid=0 2024-11-21T03:25:50,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742288_1464 (size=12301) 2024-11-21T03:25:50,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/771a15626db24f9ea6d1cff106502977 2024-11-21T03:25:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/1814ff7adcf44af388b06d3e386f8e18 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1814ff7adcf44af388b06d3e386f8e18 2024-11-21T03:25:50,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1814ff7adcf44af388b06d3e386f8e18, entries=200, sequenceid=327, filesize=39.0 K 2024-11-21T03:25:50,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/e2dc396c1d874723a0f9947d8267859f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e2dc396c1d874723a0f9947d8267859f 2024-11-21T03:25:50,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e2dc396c1d874723a0f9947d8267859f, entries=150, sequenceid=327, filesize=12.0 K 2024-11-21T03:25:50,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/771a15626db24f9ea6d1cff106502977 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/771a15626db24f9ea6d1cff106502977 2024-11-21T03:25:50,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/771a15626db24f9ea6d1cff106502977, entries=150, sequenceid=327, filesize=12.0 K 2024-11-21T03:25:50,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for cc2795dfb3064428041463dd8455a05e in 901ms, sequenceid=327, compaction requested=true 2024-11-21T03:25:50,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:50,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:25:50,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:50,647 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:50,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:25:50,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:50,647 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:50,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc2795dfb3064428041463dd8455a05e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:25:50,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:50,648 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:50,648 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:50,648 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/B is initiating minor compaction (all files) 2024-11-21T03:25:50,648 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/A is initiating minor compaction (all files) 2024-11-21T03:25:50,648 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/A in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:50,648 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/B in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:50,648 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/a8cc6300c49c474ca2e932de766f5126, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6147915c572c4289aaa00800e11663e2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/edbaaf7dc4d34d978e7545ee02c0a4d9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e2dc396c1d874723a0f9947d8267859f] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=48.7 K 2024-11-21T03:25:50,648 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/61a03151f7d14803bb769fa82ddc54ff, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/18559659865741b9994cb182d5d1fee5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/fbab863df576466590f6dfb08b474a2a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1814ff7adcf44af388b06d3e386f8e18] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=139.7 K 2024-11-21T03:25:50,648 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:50,648 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/61a03151f7d14803bb769fa82ddc54ff, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/18559659865741b9994cb182d5d1fee5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/fbab863df576466590f6dfb08b474a2a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1814ff7adcf44af388b06d3e386f8e18] 2024-11-21T03:25:50,649 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting a8cc6300c49c474ca2e932de766f5126, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159545336 2024-11-21T03:25:50,649 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61a03151f7d14803bb769fa82ddc54ff, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159545336 2024-11-21T03:25:50,649 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6147915c572c4289aaa00800e11663e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732159545797 2024-11-21T03:25:50,649 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18559659865741b9994cb182d5d1fee5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732159545797 2024-11-21T03:25:50,649 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting edbaaf7dc4d34d978e7545ee02c0a4d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732159547064 2024-11-21T03:25:50,649 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbab863df576466590f6dfb08b474a2a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732159547064 2024-11-21T03:25:50,649 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e2dc396c1d874723a0f9947d8267859f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732159549407 2024-11-21T03:25:50,649 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1814ff7adcf44af388b06d3e386f8e18, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732159549399 2024-11-21T03:25:50,661 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:50,669 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112120d60cf4a5f9453eb112de0acf1dc2b3_cc2795dfb3064428041463dd8455a05e store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:50,671 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112120d60cf4a5f9453eb112de0acf1dc2b3_cc2795dfb3064428041463dd8455a05e, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:50,671 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112120d60cf4a5f9453eb112de0acf1dc2b3_cc2795dfb3064428041463dd8455a05e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:50,674 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:50,674 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/25bdcff3b40840a0876a5bc1a5b12cb3 is 50, key is test_row_0/B:col10/1732159549415/Put/seqid=0 2024-11-21T03:25:50,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742289_1465 (size=4469) 2024-11-21T03:25:50,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742290_1466 (size=13085) 2024-11-21T03:25:50,703 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/25bdcff3b40840a0876a5bc1a5b12cb3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/25bdcff3b40840a0876a5bc1a5b12cb3 2024-11-21T03:25:50,709 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc2795dfb3064428041463dd8455a05e/B of cc2795dfb3064428041463dd8455a05e into 25bdcff3b40840a0876a5bc1a5b12cb3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:50,709 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:50,709 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/B, priority=12, startTime=1732159550647; duration=0sec 2024-11-21T03:25:50,709 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:25:50,709 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:B 2024-11-21T03:25:50,710 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:25:50,712 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:25:50,712 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): cc2795dfb3064428041463dd8455a05e/C is initiating minor compaction (all files) 2024-11-21T03:25:50,712 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc2795dfb3064428041463dd8455a05e/C in TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:50,712 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/face34f18ffd4aafa491dde5063ad6d0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f406b87bdb0240329f13122974fd473b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/4356ae29adef4b5695cadf1738f889fb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/771a15626db24f9ea6d1cff106502977] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp, totalSize=48.7 K 2024-11-21T03:25:50,713 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting face34f18ffd4aafa491dde5063ad6d0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159545336 2024-11-21T03:25:50,713 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting f406b87bdb0240329f13122974fd473b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732159545797 2024-11-21T03:25:50,713 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4356ae29adef4b5695cadf1738f889fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732159547064 2024-11-21T03:25:50,715 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 771a15626db24f9ea6d1cff106502977, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732159549407 2024-11-21T03:25:50,736 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#C#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:50,737 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/e5bc496de307446480e9ebd42f6cd04a is 50, key is test_row_0/C:col10/1732159549415/Put/seqid=0 2024-11-21T03:25:50,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742291_1467 (size=13085) 2024-11-21T03:25:50,948 INFO [Thread-1688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1688. 2024-11-21T03:25:50,948 DEBUG [Thread-1688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:50,948 DEBUG [Thread-1688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,948 DEBUG [Thread-1688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:50,950 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:25:50,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:25:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:50,952 INFO [Thread-1681 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1681. 2024-11-21T03:25:50,952 DEBUG [Thread-1681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:50,952 DEBUG [Thread-1681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,952 DEBUG [Thread-1681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,953 INFO [Thread-1694 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1694. 2024-11-21T03:25:50,953 DEBUG [Thread-1694 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:50,953 DEBUG [Thread-1694 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,953 DEBUG [Thread-1694 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:50,955 INFO [Thread-1696 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1696. 2024-11-21T03:25:50,955 DEBUG [Thread-1696 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:50,955 DEBUG [Thread-1696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,956 DEBUG [Thread-1696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:50,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:50,963 INFO [Thread-1690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1690. 2024-11-21T03:25:50,963 DEBUG [Thread-1690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:50,963 DEBUG [Thread-1690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,963 INFO [Thread-1692 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1692. 2024-11-21T03:25:50,963 DEBUG [Thread-1692 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:50,963 DEBUG [Thread-1692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,963 DEBUG [Thread-1690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,964 DEBUG [Thread-1692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,964 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:50,964 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:50,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d3dbe7a5e39143db858d8922202f157e_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_0/A:col10/1732159549794/Put/seqid=0 2024-11-21T03:25:50,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742292_1468 (size=14994) 2024-11-21T03:25:50,974 INFO [Thread-1685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1685. 2024-11-21T03:25:50,974 DEBUG [Thread-1685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:50,974 DEBUG [Thread-1685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,974 DEBUG [Thread-1685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:50,989 INFO [Thread-1677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1677. 2024-11-21T03:25:50,989 DEBUG [Thread-1677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:50,989 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,989 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:50,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:51,084 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc2795dfb3064428041463dd8455a05e#A#compaction#390 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:25:51,084 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/583cd93e5f024a9fbc13c8861074b111 is 175, key is test_row_0/A:col10/1732159549415/Put/seqid=0 2024-11-21T03:25:51,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742293_1469 (size=32039) 2024-11-21T03:25:51,098 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/583cd93e5f024a9fbc13c8861074b111 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/583cd93e5f024a9fbc13c8861074b111 2024-11-21T03:25:51,103 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc2795dfb3064428041463dd8455a05e/A of cc2795dfb3064428041463dd8455a05e into 583cd93e5f024a9fbc13c8861074b111(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:51,103 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:51,103 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/A, priority=12, startTime=1732159550647; duration=0sec 2024-11-21T03:25:51,103 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:51,103 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:A 2024-11-21T03:25:51,159 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/e5bc496de307446480e9ebd42f6cd04a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/e5bc496de307446480e9ebd42f6cd04a 2024-11-21T03:25:51,165 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc2795dfb3064428041463dd8455a05e/C of cc2795dfb3064428041463dd8455a05e into e5bc496de307446480e9ebd42f6cd04a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:25:51,165 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:51,165 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., storeName=cc2795dfb3064428041463dd8455a05e/C, priority=12, startTime=1732159550647; duration=0sec 2024-11-21T03:25:51,165 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:25:51,166 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc2795dfb3064428041463dd8455a05e:C 2024-11-21T03:25:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T03:25:51,168 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:51,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:51,379 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d3dbe7a5e39143db858d8922202f157e_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d3dbe7a5e39143db858d8922202f157e_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:51,380 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/5eabf5048d6d4032be60ffd34fb6afdd, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:51,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/5eabf5048d6d4032be60ffd34fb6afdd is 175, key is test_row_0/A:col10/1732159549794/Put/seqid=0 2024-11-21T03:25:51,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742294_1470 (size=39949) 2024-11-21T03:25:51,389 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/5eabf5048d6d4032be60ffd34fb6afdd 2024-11-21T03:25:51,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/167133d0bf9e4ef59fec61919715c9ce is 50, key is test_row_0/B:col10/1732159549794/Put/seqid=0 2024-11-21T03:25:51,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742295_1471 (size=12301) 2024-11-21T03:25:51,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/167133d0bf9e4ef59fec61919715c9ce 2024-11-21T03:25:51,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/01e2dcfce61e44f1a1ca7bad3fb4a02c is 50, key is test_row_0/C:col10/1732159549794/Put/seqid=0 2024-11-21T03:25:51,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742296_1472 (size=12301) 2024-11-21T03:25:52,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/01e2dcfce61e44f1a1ca7bad3fb4a02c 2024-11-21T03:25:52,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/5eabf5048d6d4032be60ffd34fb6afdd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5eabf5048d6d4032be60ffd34fb6afdd 2024-11-21T03:25:52,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5eabf5048d6d4032be60ffd34fb6afdd, entries=200, sequenceid=340, filesize=39.0 K 2024-11-21T03:25:52,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/167133d0bf9e4ef59fec61919715c9ce as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/167133d0bf9e4ef59fec61919715c9ce 2024-11-21T03:25:52,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/167133d0bf9e4ef59fec61919715c9ce, entries=150, sequenceid=340, filesize=12.0 K 2024-11-21T03:25:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/01e2dcfce61e44f1a1ca7bad3fb4a02c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/01e2dcfce61e44f1a1ca7bad3fb4a02c 2024-11-21T03:25:52,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/01e2dcfce61e44f1a1ca7bad3fb4a02c, entries=150, sequenceid=340, filesize=12.0 K 2024-11-21T03:25:52,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=13.42 KB/13740 for cc2795dfb3064428041463dd8455a05e in 1301ms, sequenceid=340, compaction requested=false 2024-11-21T03:25:52,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc2795dfb3064428041463dd8455a05e: 2024-11-21T03:25:53,589 INFO [Thread-1683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1683. 2024-11-21T03:25:53,589 DEBUG [Thread-1683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:53,589 DEBUG [Thread-1683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:53,589 DEBUG [Thread-1683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:53,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:57,630 INFO [Thread-1679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1679. 2024-11-21T03:25:57,630 DEBUG [Thread-1679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:25:57,630 DEBUG [Thread-1679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:57,631 DEBUG [Thread-1679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:57,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:57,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T03:25:57,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-21T03:25:57,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-21T03:25:57,631 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 872 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2616 rows 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 871 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2613 rows 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 870 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2610 rows 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 879 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2637 rows 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 877 2024-11-21T03:25:57,632 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2630 rows 2024-11-21T03:25:57,632 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:25:57,633 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:25:57,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:57,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:25:57,633 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:25:57,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T03:25:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T03:25:57,641 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159557640"}]},"ts":"1732159557640"} 2024-11-21T03:25:57,642 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T03:25:57,643 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T03:25:57,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:25:57,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, UNASSIGN}] 2024-11-21T03:25:57,646 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, UNASSIGN 2024-11-21T03:25:57,647 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=cc2795dfb3064428041463dd8455a05e, regionState=CLOSING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:25:57,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, UNASSIGN because future has completed 2024-11-21T03:25:57,648 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:25:57,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:25:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T03:25:57,801 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:57,801 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:25:57,802 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing cc2795dfb3064428041463dd8455a05e, disabling compactions & flushes 2024-11-21T03:25:57,802 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:57,802 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:57,802 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. after waiting 0 ms 2024-11-21T03:25:57,802 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:57,802 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing cc2795dfb3064428041463dd8455a05e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-21T03:25:57,809 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e799366c016a471ba73478970653b13b_cc2795dfb3064428041463dd8455a05e is 50, key is test_row_1/A:col10/1732159550988/Put/seqid=0 2024-11-21T03:25:57,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742297_1473 (size=9914) 2024-11-21T03:25:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T03:25:58,214 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:25:58,217 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e799366c016a471ba73478970653b13b_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e799366c016a471ba73478970653b13b_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:58,218 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/b465eadf6e8647219570fd8a39d3c540, store: [table=TestAcidGuarantees family=A region=cc2795dfb3064428041463dd8455a05e] 2024-11-21T03:25:58,218 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/b465eadf6e8647219570fd8a39d3c540 is 175, key is test_row_1/A:col10/1732159550988/Put/seqid=0 2024-11-21T03:25:58,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742298_1474 (size=22561) 2024-11-21T03:25:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T03:25:58,622 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/b465eadf6e8647219570fd8a39d3c540 2024-11-21T03:25:58,630 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/78a189ab9ded4e3d903c1995b9ca573e is 50, key is test_row_1/B:col10/1732159550988/Put/seqid=0 2024-11-21T03:25:58,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742299_1475 (size=9857) 2024-11-21T03:25:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T03:25:59,034 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/78a189ab9ded4e3d903c1995b9ca573e 2024-11-21T03:25:59,040 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/2066387c3ee44f3b83289b22441cd7f4 is 50, key is test_row_1/C:col10/1732159550988/Put/seqid=0 2024-11-21T03:25:59,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742300_1476 (size=9857) 2024-11-21T03:25:59,317 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T03:25:59,444 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/2066387c3ee44f3b83289b22441cd7f4 2024-11-21T03:25:59,449 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/A/b465eadf6e8647219570fd8a39d3c540 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b465eadf6e8647219570fd8a39d3c540 2024-11-21T03:25:59,453 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b465eadf6e8647219570fd8a39d3c540, entries=100, sequenceid=349, filesize=22.0 K 2024-11-21T03:25:59,454 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/B/78a189ab9ded4e3d903c1995b9ca573e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/78a189ab9ded4e3d903c1995b9ca573e 2024-11-21T03:25:59,459 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/78a189ab9ded4e3d903c1995b9ca573e, entries=100, sequenceid=349, filesize=9.6 K 2024-11-21T03:25:59,460 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/.tmp/C/2066387c3ee44f3b83289b22441cd7f4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/2066387c3ee44f3b83289b22441cd7f4 2024-11-21T03:25:59,466 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/2066387c3ee44f3b83289b22441cd7f4, entries=100, sequenceid=349, filesize=9.6 K 2024-11-21T03:25:59,467 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for cc2795dfb3064428041463dd8455a05e in 1665ms, sequenceid=349, compaction requested=true 2024-11-21T03:25:59,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b00ae7a7fcfe452d91ec66159963f89a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/bb0bfcebbafb4802853568a8a6bdcdc6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/11f932732c9c473cb42cdecb468af635, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1b4b54b3ba394396aa052fbb4aa74041, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/647474e4fe174948adaa1c1cc4a56195, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/82d3dcfef8094165b5b0dae8f1d1086c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1100038e67d14e2f9751bf401d2cd612, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/354bb55e92894875b1296df228afb33c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/7ca60bd20f1f43139b1f6876662b84b9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/cf837901b3f741f384118496e0717786, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/8c05328d9acd407ab1d95253a1882631, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/9927be30e71f46cc9a4b4fcbde58f073, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/272c792cf45c4d7692844205e973f032, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3fcbb41a180d4b2c9ec93718db9b285b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3263d22f2f254943a24a21144706971f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/25743259578e4cf28fbf75e0cb24c97c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/53b6c1f180d64ef59e0410167b00815e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/2e85974472c54e4f82696092431a4711, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/61a03151f7d14803bb769fa82ddc54ff, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5e9284746f4843fe997821143fa2f5f8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/18559659865741b9994cb182d5d1fee5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/fbab863df576466590f6dfb08b474a2a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1814ff7adcf44af388b06d3e386f8e18] to archive 2024-11-21T03:25:59,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:25:59,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b00ae7a7fcfe452d91ec66159963f89a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b00ae7a7fcfe452d91ec66159963f89a 2024-11-21T03:25:59,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/bb0bfcebbafb4802853568a8a6bdcdc6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/bb0bfcebbafb4802853568a8a6bdcdc6 2024-11-21T03:25:59,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/11f932732c9c473cb42cdecb468af635 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/11f932732c9c473cb42cdecb468af635 2024-11-21T03:25:59,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1b4b54b3ba394396aa052fbb4aa74041 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1b4b54b3ba394396aa052fbb4aa74041 2024-11-21T03:25:59,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/647474e4fe174948adaa1c1cc4a56195 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/647474e4fe174948adaa1c1cc4a56195 2024-11-21T03:25:59,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/82d3dcfef8094165b5b0dae8f1d1086c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/82d3dcfef8094165b5b0dae8f1d1086c 2024-11-21T03:25:59,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1100038e67d14e2f9751bf401d2cd612 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1100038e67d14e2f9751bf401d2cd612 2024-11-21T03:25:59,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/354bb55e92894875b1296df228afb33c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/354bb55e92894875b1296df228afb33c 2024-11-21T03:25:59,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/7ca60bd20f1f43139b1f6876662b84b9 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/7ca60bd20f1f43139b1f6876662b84b9 2024-11-21T03:25:59,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/cf837901b3f741f384118496e0717786 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/cf837901b3f741f384118496e0717786 2024-11-21T03:25:59,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/8c05328d9acd407ab1d95253a1882631 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/8c05328d9acd407ab1d95253a1882631 2024-11-21T03:25:59,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/9927be30e71f46cc9a4b4fcbde58f073 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/9927be30e71f46cc9a4b4fcbde58f073 2024-11-21T03:25:59,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/272c792cf45c4d7692844205e973f032 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/272c792cf45c4d7692844205e973f032 2024-11-21T03:25:59,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3fcbb41a180d4b2c9ec93718db9b285b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3fcbb41a180d4b2c9ec93718db9b285b 2024-11-21T03:25:59,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3263d22f2f254943a24a21144706971f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/3263d22f2f254943a24a21144706971f 2024-11-21T03:25:59,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/25743259578e4cf28fbf75e0cb24c97c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/25743259578e4cf28fbf75e0cb24c97c 2024-11-21T03:25:59,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/53b6c1f180d64ef59e0410167b00815e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/53b6c1f180d64ef59e0410167b00815e 2024-11-21T03:25:59,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/2e85974472c54e4f82696092431a4711 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/2e85974472c54e4f82696092431a4711 2024-11-21T03:25:59,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/61a03151f7d14803bb769fa82ddc54ff to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/61a03151f7d14803bb769fa82ddc54ff 2024-11-21T03:25:59,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5e9284746f4843fe997821143fa2f5f8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5e9284746f4843fe997821143fa2f5f8 2024-11-21T03:25:59,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/18559659865741b9994cb182d5d1fee5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/18559659865741b9994cb182d5d1fee5 2024-11-21T03:25:59,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/fbab863df576466590f6dfb08b474a2a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/fbab863df576466590f6dfb08b474a2a 2024-11-21T03:25:59,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1814ff7adcf44af388b06d3e386f8e18 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/1814ff7adcf44af388b06d3e386f8e18 2024-11-21T03:25:59,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1a7b524780ec4401ba3663169f74455b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6da1602c98334a5896e2e30f578786fc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/faeb4f98d1be4aa5a4aa41da12efcfeb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/d2feda72e3c049dfafbe64deb072c6ce, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/17285a2540d742caaaf8771f8c73be5a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/77fa12fa4f10431291bca65f3df36da3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/4e76ba7483ca47e396f8c4c0acffa6f4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/337637a1f186441b8f42e7b5810c99f7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/b9434db9f838464a8c446c0afc348eb1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e495d84813304f3687e42a2f93465e8c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/96bc83d094424b49bf9e70db4648f61b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1f1c7770c9574687a1cdcbfa08e52254, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/2ba3c71aab0442fe842ca41d2d05268e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/f68e6bb15fd14644bff689b9de02cb57, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/af3f636bd3e44320ba3d2b28710a10a3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/10e1b02ae59e427cb391c985fd7f7c67, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1e4c50a783744419a565acf91cd96c61, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6675e4547dd6467494a3bbfd894ed30f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/a8cc6300c49c474ca2e932de766f5126, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/24d5f13da16f4154b2e105f15d3f985b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6147915c572c4289aaa00800e11663e2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/edbaaf7dc4d34d978e7545ee02c0a4d9, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e2dc396c1d874723a0f9947d8267859f] to archive 2024-11-21T03:25:59,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:25:59,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1a7b524780ec4401ba3663169f74455b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1a7b524780ec4401ba3663169f74455b 2024-11-21T03:25:59,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6da1602c98334a5896e2e30f578786fc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6da1602c98334a5896e2e30f578786fc 2024-11-21T03:25:59,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/faeb4f98d1be4aa5a4aa41da12efcfeb to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/faeb4f98d1be4aa5a4aa41da12efcfeb 2024-11-21T03:25:59,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/d2feda72e3c049dfafbe64deb072c6ce to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/d2feda72e3c049dfafbe64deb072c6ce 2024-11-21T03:25:59,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/17285a2540d742caaaf8771f8c73be5a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/17285a2540d742caaaf8771f8c73be5a 2024-11-21T03:25:59,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/77fa12fa4f10431291bca65f3df36da3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/77fa12fa4f10431291bca65f3df36da3 2024-11-21T03:25:59,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/4e76ba7483ca47e396f8c4c0acffa6f4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/4e76ba7483ca47e396f8c4c0acffa6f4 2024-11-21T03:25:59,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/337637a1f186441b8f42e7b5810c99f7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/337637a1f186441b8f42e7b5810c99f7 2024-11-21T03:25:59,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/b9434db9f838464a8c446c0afc348eb1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/b9434db9f838464a8c446c0afc348eb1 2024-11-21T03:25:59,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e495d84813304f3687e42a2f93465e8c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e495d84813304f3687e42a2f93465e8c 2024-11-21T03:25:59,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/96bc83d094424b49bf9e70db4648f61b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/96bc83d094424b49bf9e70db4648f61b 2024-11-21T03:25:59,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1f1c7770c9574687a1cdcbfa08e52254 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1f1c7770c9574687a1cdcbfa08e52254 2024-11-21T03:25:59,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/2ba3c71aab0442fe842ca41d2d05268e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/2ba3c71aab0442fe842ca41d2d05268e 2024-11-21T03:25:59,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/f68e6bb15fd14644bff689b9de02cb57 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/f68e6bb15fd14644bff689b9de02cb57 2024-11-21T03:25:59,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/af3f636bd3e44320ba3d2b28710a10a3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/af3f636bd3e44320ba3d2b28710a10a3 2024-11-21T03:25:59,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/10e1b02ae59e427cb391c985fd7f7c67 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/10e1b02ae59e427cb391c985fd7f7c67 2024-11-21T03:25:59,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1e4c50a783744419a565acf91cd96c61 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/1e4c50a783744419a565acf91cd96c61 2024-11-21T03:25:59,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6675e4547dd6467494a3bbfd894ed30f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6675e4547dd6467494a3bbfd894ed30f 2024-11-21T03:25:59,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/a8cc6300c49c474ca2e932de766f5126 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/a8cc6300c49c474ca2e932de766f5126 2024-11-21T03:25:59,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/24d5f13da16f4154b2e105f15d3f985b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/24d5f13da16f4154b2e105f15d3f985b 2024-11-21T03:25:59,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6147915c572c4289aaa00800e11663e2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/6147915c572c4289aaa00800e11663e2 2024-11-21T03:25:59,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/edbaaf7dc4d34d978e7545ee02c0a4d9 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/edbaaf7dc4d34d978e7545ee02c0a4d9 2024-11-21T03:25:59,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e2dc396c1d874723a0f9947d8267859f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/e2dc396c1d874723a0f9947d8267859f 2024-11-21T03:25:59,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/1d8ca631b9fa4e1baa95f4db3614b2cc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/322ee2d5a22b400aace4ca2f34653dc6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/488efb1b10644008a9c7261739aa9d17, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/44d3629ed60f49b5bd8c9c28bf2b0ad8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f2b24691ec7e4b3b84e1b8321e4eb3ec, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/9a91967eecda44a0b1e3f6d1921ddc1d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/fbf20d9eb5e34f40b6d905ebcee01e42, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/36bad9ea7b724a4ea07e0aa6beb856ce, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/357aaf115ff94bb8a780847191efcadf, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/b09ee8e56bfc466badd11d9987ba2033, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/be5f668c133145449b9419aa874fdd9a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/aacbbf56954d4cc7aaac106ad12719f7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/477ce5b6c1284d8688c9ad13985f9387, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/5b7ddf7749cc4dd6b61ea884bf743417, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/736b53915d52468bbfc5fb79e74e918a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7423ce0852784335a0ac2503da408925, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/46d9834ed5df4a5aaea9c977ab7c6641, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7986264473e9497fbd7a786885fde979, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/face34f18ffd4aafa491dde5063ad6d0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7813694448ae414cb5ecf88a892c2667, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f406b87bdb0240329f13122974fd473b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/4356ae29adef4b5695cadf1738f889fb, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/771a15626db24f9ea6d1cff106502977] to archive 2024-11-21T03:25:59,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:25:59,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/1d8ca631b9fa4e1baa95f4db3614b2cc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/1d8ca631b9fa4e1baa95f4db3614b2cc 2024-11-21T03:25:59,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/322ee2d5a22b400aace4ca2f34653dc6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/322ee2d5a22b400aace4ca2f34653dc6 2024-11-21T03:25:59,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/488efb1b10644008a9c7261739aa9d17 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/488efb1b10644008a9c7261739aa9d17 2024-11-21T03:25:59,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/44d3629ed60f49b5bd8c9c28bf2b0ad8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/44d3629ed60f49b5bd8c9c28bf2b0ad8 2024-11-21T03:25:59,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f2b24691ec7e4b3b84e1b8321e4eb3ec to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f2b24691ec7e4b3b84e1b8321e4eb3ec 2024-11-21T03:25:59,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/9a91967eecda44a0b1e3f6d1921ddc1d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/9a91967eecda44a0b1e3f6d1921ddc1d 2024-11-21T03:25:59,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/fbf20d9eb5e34f40b6d905ebcee01e42 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/fbf20d9eb5e34f40b6d905ebcee01e42 2024-11-21T03:25:59,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/36bad9ea7b724a4ea07e0aa6beb856ce to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/36bad9ea7b724a4ea07e0aa6beb856ce 2024-11-21T03:25:59,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/357aaf115ff94bb8a780847191efcadf to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/357aaf115ff94bb8a780847191efcadf 2024-11-21T03:25:59,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/b09ee8e56bfc466badd11d9987ba2033 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/b09ee8e56bfc466badd11d9987ba2033 2024-11-21T03:25:59,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/be5f668c133145449b9419aa874fdd9a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/be5f668c133145449b9419aa874fdd9a 2024-11-21T03:25:59,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/aacbbf56954d4cc7aaac106ad12719f7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/aacbbf56954d4cc7aaac106ad12719f7 2024-11-21T03:25:59,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/477ce5b6c1284d8688c9ad13985f9387 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/477ce5b6c1284d8688c9ad13985f9387 2024-11-21T03:25:59,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/5b7ddf7749cc4dd6b61ea884bf743417 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/5b7ddf7749cc4dd6b61ea884bf743417 2024-11-21T03:25:59,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/736b53915d52468bbfc5fb79e74e918a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/736b53915d52468bbfc5fb79e74e918a 2024-11-21T03:25:59,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7423ce0852784335a0ac2503da408925 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7423ce0852784335a0ac2503da408925 2024-11-21T03:25:59,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/46d9834ed5df4a5aaea9c977ab7c6641 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/46d9834ed5df4a5aaea9c977ab7c6641 2024-11-21T03:25:59,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7986264473e9497fbd7a786885fde979 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7986264473e9497fbd7a786885fde979 2024-11-21T03:25:59,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/face34f18ffd4aafa491dde5063ad6d0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/face34f18ffd4aafa491dde5063ad6d0 2024-11-21T03:25:59,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7813694448ae414cb5ecf88a892c2667 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/7813694448ae414cb5ecf88a892c2667 2024-11-21T03:25:59,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f406b87bdb0240329f13122974fd473b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/f406b87bdb0240329f13122974fd473b 2024-11-21T03:25:59,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/4356ae29adef4b5695cadf1738f889fb to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/4356ae29adef4b5695cadf1738f889fb 2024-11-21T03:25:59,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/771a15626db24f9ea6d1cff106502977 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/771a15626db24f9ea6d1cff106502977 2024-11-21T03:25:59,564 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/recovered.edits/352.seqid, newMaxSeqId=352, maxSeqId=4 2024-11-21T03:25:59,565 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. 2024-11-21T03:25:59,565 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for cc2795dfb3064428041463dd8455a05e: Waiting for close lock at 1732159557801Running coprocessor pre-close hooks at 1732159557801Disabling compacts and flushes for region at 1732159557802 (+1 ms)Disabling writes for close at 1732159557802Obtaining lock to block concurrent updates at 1732159557802Preparing flush snapshotting stores in cc2795dfb3064428041463dd8455a05e at 1732159557802Finished memstore snapshotting TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732159557802Flushing stores of TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e. at 1732159557803 (+1 ms)Flushing cc2795dfb3064428041463dd8455a05e/A: creating writer at 1732159557803Flushing mob file cc2795dfb3064428041463dd8455a05e/A: appending metadata at 1732159557809 (+6 ms)Flushing mob file cc2795dfb3064428041463dd8455a05e/A: closing flushed file at 1732159557809Flushing cc2795dfb3064428041463dd8455a05e/A: appending metadata at 1732159558218 (+409 ms)Flushing cc2795dfb3064428041463dd8455a05e/A: closing flushed file at 1732159558218Flushing cc2795dfb3064428041463dd8455a05e/B: creating writer at 1732159558626 (+408 ms)Flushing cc2795dfb3064428041463dd8455a05e/B: appending metadata at 1732159558629 (+3 ms)Flushing cc2795dfb3064428041463dd8455a05e/B: closing flushed file at 1732159558629Flushing cc2795dfb3064428041463dd8455a05e/C: creating writer at 1732159559037 (+408 ms)Flushing cc2795dfb3064428041463dd8455a05e/C: appending metadata at 1732159559039 (+2 ms)Flushing cc2795dfb3064428041463dd8455a05e/C: closing flushed file at 1732159559039Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1832ab6c: reopening flushed file at 1732159559448 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1778331e: reopening flushed file at 1732159559454 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3743da68: reopening flushed file at 1732159559460 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for cc2795dfb3064428041463dd8455a05e in 1665ms, sequenceid=349, compaction requested=true at 1732159559467 (+7 ms)Writing region close event to WAL at 1732159559562 (+95 ms)Running coprocessor post-close hooks at 1732159559565 (+3 ms)Closed at 1732159559565 2024-11-21T03:25:59,567 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,568 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=cc2795dfb3064428041463dd8455a05e, regionState=CLOSED 2024-11-21T03:25:59,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:25:59,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-21T03:25:59,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure cc2795dfb3064428041463dd8455a05e, server=b69e2a1ee061,45649,1732159441548 in 1.9210 sec 2024-11-21T03:25:59,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-21T03:25:59,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc2795dfb3064428041463dd8455a05e, UNASSIGN in 1.9270 sec 2024-11-21T03:25:59,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-21T03:25:59,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9310 sec 2024-11-21T03:25:59,576 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159559576"}]},"ts":"1732159559576"} 2024-11-21T03:25:59,577 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T03:25:59,577 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T03:25:59,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9410 sec 2024-11-21T03:25:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T03:25:59,788 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:25:59,788 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T03:25:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:59,790 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T03:25:59,792 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:59,794 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,802 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/recovered.edits] 2024-11-21T03:25:59,822 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/583cd93e5f024a9fbc13c8861074b111 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/583cd93e5f024a9fbc13c8861074b111 2024-11-21T03:25:59,830 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5eabf5048d6d4032be60ffd34fb6afdd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/5eabf5048d6d4032be60ffd34fb6afdd 2024-11-21T03:25:59,837 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b465eadf6e8647219570fd8a39d3c540 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/A/b465eadf6e8647219570fd8a39d3c540 2024-11-21T03:25:59,840 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/167133d0bf9e4ef59fec61919715c9ce to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/167133d0bf9e4ef59fec61919715c9ce 2024-11-21T03:25:59,841 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/25bdcff3b40840a0876a5bc1a5b12cb3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/25bdcff3b40840a0876a5bc1a5b12cb3 2024-11-21T03:25:59,843 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/78a189ab9ded4e3d903c1995b9ca573e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/B/78a189ab9ded4e3d903c1995b9ca573e 2024-11-21T03:25:59,845 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/01e2dcfce61e44f1a1ca7bad3fb4a02c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/01e2dcfce61e44f1a1ca7bad3fb4a02c 2024-11-21T03:25:59,846 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/2066387c3ee44f3b83289b22441cd7f4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/2066387c3ee44f3b83289b22441cd7f4 2024-11-21T03:25:59,848 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/e5bc496de307446480e9ebd42f6cd04a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/C/e5bc496de307446480e9ebd42f6cd04a 2024-11-21T03:25:59,857 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/recovered.edits/352.seqid to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e/recovered.edits/352.seqid 2024-11-21T03:25:59,858 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,858 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T03:25:59,859 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T03:25:59,860 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T03:25:59,863 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210cfecd87ac7d440587d60bd79df1b4e3_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210cfecd87ac7d440587d60bd79df1b4e3_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,864 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212bc0725040a1421fb635dd73e549fbdd_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212bc0725040a1421fb635dd73e549fbdd_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,865 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215d8ad580e5b7481e987862bfdcb7a873_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215d8ad580e5b7481e987862bfdcb7a873_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,867 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112173e213f9907f4069a93489d58d53f9dc_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112173e213f9907f4069a93489d58d53f9dc_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,868 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121791873e224ff44a286122d73be539d2d_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121791873e224ff44a286122d73be539d2d_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,870 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112180e16322f3e8473fb8fd8d0bb7ffc072_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112180e16322f3e8473fb8fd8d0bb7ffc072_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,871 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182d05c821f2c48b3b02d050edf275152_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112182d05c821f2c48b3b02d050edf275152_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,872 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112183b677c0301f4cf6955dba3d8bb9f17c_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112183b677c0301f4cf6955dba3d8bb9f17c_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,874 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112188a3d7fba35345aea6ff6bd246f5b0ba_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112188a3d7fba35345aea6ff6bd246f5b0ba_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,875 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121989abf283452416389a42f2c9e06df25_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121989abf283452416389a42f2c9e06df25_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,876 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e0c3eaf03c449e6ab186b5b95b8b9bb_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e0c3eaf03c449e6ab186b5b95b8b9bb_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,878 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a2efd05e7a24487585b13db0941550cb_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a2efd05e7a24487585b13db0941550cb_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,879 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ab246234f82044d895b32eb3b1600e61_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ab246234f82044d895b32eb3b1600e61_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,880 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d006147947d042e8b5b504c129f2f634_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d006147947d042e8b5b504c129f2f634_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,881 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d3dbe7a5e39143db858d8922202f157e_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d3dbe7a5e39143db858d8922202f157e_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,883 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e13615b7e6a540dfb2e09477f435b8b9_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e13615b7e6a540dfb2e09477f435b8b9_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,884 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e2c26b8f753d44c885c8a3861d903848_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e2c26b8f753d44c885c8a3861d903848_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,885 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e6b68ccae4db4f589260aff16ee655e8_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e6b68ccae4db4f589260aff16ee655e8_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,887 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e799366c016a471ba73478970653b13b_cc2795dfb3064428041463dd8455a05e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e799366c016a471ba73478970653b13b_cc2795dfb3064428041463dd8455a05e 2024-11-21T03:25:59,887 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T03:25:59,890 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:59,894 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T03:25:59,896 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T03:25:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T03:25:59,897 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:59,898 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T03:25:59,898 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732159559898"}]},"ts":"9223372036854775807"} 2024-11-21T03:25:59,900 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T03:25:59,900 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cc2795dfb3064428041463dd8455a05e, NAME => 'TestAcidGuarantees,,1732159527689.cc2795dfb3064428041463dd8455a05e.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T03:25:59,900 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T03:25:59,900 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732159559900"}]},"ts":"9223372036854775807"} 2024-11-21T03:25:59,902 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T03:25:59,903 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:25:59,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 115 msec 2024-11-21T03:26:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T03:26:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T03:26:00,116 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:00,126 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 241), OpenFileDescriptor=453 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1045 (was 981) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1834 (was 2236) 2024-11-21T03:26:00,139 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1045, ProcessCount=11, AvailableMemoryMB=1834 2024-11-21T03:26:00,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:26:00,141 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:26:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:00,145 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:26:00,145 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:00,145 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-21T03:26:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-21T03:26:00,146 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:26:00,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742301_1477 (size=959) 2024-11-21T03:26:00,155 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1 2024-11-21T03:26:00,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742302_1478 (size=53) 2024-11-21T03:26:00,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:26:00,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 215fc242d64dec572e979318d587db2b, disabling compactions & flushes 2024-11-21T03:26:00,166 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:00,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:00,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. after waiting 0 ms 2024-11-21T03:26:00,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:00,166 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:00,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 215fc242d64dec572e979318d587db2b: Waiting for close lock at 1732159560166Disabling compacts and flushes for region at 1732159560166Disabling writes for close at 1732159560166Writing region close event to WAL at 1732159560166Closed at 1732159560166 2024-11-21T03:26:00,167 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:26:00,167 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732159560167"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732159560167"}]},"ts":"1732159560167"} 2024-11-21T03:26:00,169 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:26:00,171 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:26:00,171 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159560171"}]},"ts":"1732159560171"} 2024-11-21T03:26:00,173 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:26:00,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=215fc242d64dec572e979318d587db2b, ASSIGN}] 2024-11-21T03:26:00,175 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=215fc242d64dec572e979318d587db2b, ASSIGN 2024-11-21T03:26:00,176 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=215fc242d64dec572e979318d587db2b, ASSIGN; state=OFFLINE, location=b69e2a1ee061,45649,1732159441548; forceNewPlan=false, retain=false 2024-11-21T03:26:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-21T03:26:00,326 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=215fc242d64dec572e979318d587db2b, regionState=OPENING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:00,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=215fc242d64dec572e979318d587db2b, ASSIGN because future has completed 2024-11-21T03:26:00,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:26:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-21T03:26:00,486 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:00,487 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:26:00,488 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,489 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:26:00,489 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,489 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,490 INFO [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,492 INFO [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 215fc242d64dec572e979318d587db2b columnFamilyName A 2024-11-21T03:26:00,492 DEBUG [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:00,494 INFO [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] regionserver.HStore(327): Store=215fc242d64dec572e979318d587db2b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:26:00,495 INFO [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,496 INFO [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 215fc242d64dec572e979318d587db2b columnFamilyName B 2024-11-21T03:26:00,496 DEBUG [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:00,497 INFO [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] regionserver.HStore(327): Store=215fc242d64dec572e979318d587db2b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:26:00,497 INFO [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,500 INFO [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 215fc242d64dec572e979318d587db2b columnFamilyName C 2024-11-21T03:26:00,500 DEBUG [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:00,501 INFO [StoreOpener-215fc242d64dec572e979318d587db2b-1 {}] regionserver.HStore(327): Store=215fc242d64dec572e979318d587db2b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:26:00,501 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,502 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,502 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,503 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,503 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,504 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:26:00,505 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,506 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:26:00,507 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened 215fc242d64dec572e979318d587db2b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70292364, jitterRate=0.047437846660614014}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:26:00,507 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,507 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for 215fc242d64dec572e979318d587db2b: Running coprocessor pre-open hook at 1732159560489Writing region info on filesystem at 1732159560489Initializing all the Stores at 1732159560490 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159560490Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159560490Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159560490Cleaning up temporary data from old regions at 1732159560503 (+13 ms)Running coprocessor post-open hooks at 1732159560507 (+4 ms)Region opened successfully at 1732159560507 2024-11-21T03:26:00,508 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., pid=110, masterSystemTime=1732159560480 2024-11-21T03:26:00,509 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:00,509 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:00,510 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=215fc242d64dec572e979318d587db2b, regionState=OPEN, openSeqNum=2, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:00,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:26:00,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-21T03:26:00,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure 215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 in 184 msec 2024-11-21T03:26:00,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-21T03:26:00,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=215fc242d64dec572e979318d587db2b, ASSIGN in 340 msec 2024-11-21T03:26:00,517 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:26:00,517 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159560517"}]},"ts":"1732159560517"} 2024-11-21T03:26:00,519 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:26:00,520 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:26:00,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 379 msec 2024-11-21T03:26:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-21T03:26:00,768 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:00,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,771 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c37c862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,774 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49de5ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,775 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,778 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:26:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is b69e2a1ee061,42023,1732159440464 2024-11-21T03:26:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3eba8074 2024-11-21T03:26:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:26:00,780 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:26:00,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,782 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1885c451, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,784 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f4043f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,787 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41a77d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,788 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a2bbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,790 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ebae51d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,792 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac56c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,793 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d561c33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,795 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c677dcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,796 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cae5408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,797 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6631ac36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,799 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76ae9b7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,800 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe1560d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,801 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c3ac43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,802 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51bdcda2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,804 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7083b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,805 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f312c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,806 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,807 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e32ccf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,808 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6bdc58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:00,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:00,810 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4867cb49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:00,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:00,811 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7767a7d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:00,812 DEBUG [Thread-2147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,812 DEBUG [Thread-2136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,812 DEBUG [Thread-2140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,812 DEBUG [Thread-2151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,812 DEBUG [Thread-2149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,813 DEBUG [Thread-2134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,813 DEBUG [Thread-2132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,814 DEBUG [Thread-2138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:26:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:26:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:26:00,817 DEBUG [Thread-2143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,817 DEBUG [Thread-2145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:26:00,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,818 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:26:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-21T03:26:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:26:00,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:26:00,820 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:00,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:26:00,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:00,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:26:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:26:00,827 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:00,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2] 2024-11-21T03:26:00,837 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:00,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:26:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:00,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/642116801803413c9fbdd483f86d4d36 is 50, key is test_row_0/A:col10/1732159560824/Put/seqid=0 2024-11-21T03:26:00,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742303_1479 (size=12001) 2024-11-21T03:26:00,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/642116801803413c9fbdd483f86d4d36 2024-11-21T03:26:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:26:00,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/f22cdff474c1476da7c4985d2e850f4a is 50, key is test_row_0/B:col10/1732159560824/Put/seqid=0 2024-11-21T03:26:00,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159620934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:00,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159620935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:00,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159620937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:00,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159620953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:00,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:00,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159620957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:00,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742304_1480 (size=12001) 2024-11-21T03:26:00,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T03:26:00,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:00,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:00,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:00,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:00,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159621059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159621069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159621069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159621069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159621087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T03:26:01,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:01,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:26:01,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159621267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T03:26:01,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:01,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159621293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159621293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159621297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159621297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/f22cdff474c1476da7c4985d2e850f4a 2024-11-21T03:26:01,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T03:26:01,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:01,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:26:01,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/fcb33bff28a54e9eaf906f22229d1267 is 50, key is test_row_0/C:col10/1732159560824/Put/seqid=0 2024-11-21T03:26:01,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742305_1481 (size=12001) 2024-11-21T03:26:01,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159621588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159621607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159621607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159621607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159621608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:01,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T03:26:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:01,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T03:26:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T03:26:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:01,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:01,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/fcb33bff28a54e9eaf906f22229d1267 2024-11-21T03:26:01,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/642116801803413c9fbdd483f86d4d36 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/642116801803413c9fbdd483f86d4d36 2024-11-21T03:26:01,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/642116801803413c9fbdd483f86d4d36, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T03:26:01,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/f22cdff474c1476da7c4985d2e850f4a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/f22cdff474c1476da7c4985d2e850f4a 2024-11-21T03:26:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:26:01,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/f22cdff474c1476da7c4985d2e850f4a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T03:26:01,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/fcb33bff28a54e9eaf906f22229d1267 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fcb33bff28a54e9eaf906f22229d1267 2024-11-21T03:26:01,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fcb33bff28a54e9eaf906f22229d1267, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T03:26:01,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 215fc242d64dec572e979318d587db2b in 1149ms, sequenceid=13, compaction requested=false 2024-11-21T03:26:01,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:02,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T03:26:02,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:02,085 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:26:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:02,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:02,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/1f0d93da9f6745418ecd111fefcfbddd is 50, key is test_row_0/A:col10/1732159560889/Put/seqid=0 2024-11-21T03:26:02,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742306_1482 (size=12001) 2024-11-21T03:26:02,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159622132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159622133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159622132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159622153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159622161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159622257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159622258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159622258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159622272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159622277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159622478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159622478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159622478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159622487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159622487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,533 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/1f0d93da9f6745418ecd111fefcfbddd 2024-11-21T03:26:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c747bf6c55f0476c9ce09e1de1bffd4d is 50, key is test_row_0/B:col10/1732159560889/Put/seqid=0 2024-11-21T03:26:02,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742307_1483 (size=12001) 2024-11-21T03:26:02,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159622787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159622788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159622788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159622798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159622798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:02,977 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c747bf6c55f0476c9ce09e1de1bffd4d 2024-11-21T03:26:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:26:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/cd028d5b8fff44a8b0a7a2eb05006001 is 50, key is test_row_0/C:col10/1732159560889/Put/seqid=0 2024-11-21T03:26:03,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742308_1484 (size=12001) 2024-11-21T03:26:03,071 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/cd028d5b8fff44a8b0a7a2eb05006001 2024-11-21T03:26:03,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/1f0d93da9f6745418ecd111fefcfbddd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f0d93da9f6745418ecd111fefcfbddd 2024-11-21T03:26:03,079 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f0d93da9f6745418ecd111fefcfbddd, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:26:03,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c747bf6c55f0476c9ce09e1de1bffd4d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c747bf6c55f0476c9ce09e1de1bffd4d 2024-11-21T03:26:03,087 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c747bf6c55f0476c9ce09e1de1bffd4d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:26:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/cd028d5b8fff44a8b0a7a2eb05006001 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/cd028d5b8fff44a8b0a7a2eb05006001 2024-11-21T03:26:03,091 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/cd028d5b8fff44a8b0a7a2eb05006001, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:26:03,092 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 215fc242d64dec572e979318d587db2b in 1007ms, sequenceid=37, compaction requested=false 2024-11-21T03:26:03,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:03,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:03,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-21T03:26:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-21T03:26:03,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-21T03:26:03,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2660 sec 2024-11-21T03:26:03,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.2790 sec 2024-11-21T03:26:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:03,301 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:03,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:26:03,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/a81599aa4f7b4880875222247de618bd is 50, key is test_row_0/A:col10/1732159562130/Put/seqid=0 2024-11-21T03:26:03,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742309_1485 (size=14341) 2024-11-21T03:26:03,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/a81599aa4f7b4880875222247de618bd 2024-11-21T03:26:03,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/93f6fc8e43844297bdfcf85b92396cb2 is 50, key is test_row_0/B:col10/1732159562130/Put/seqid=0 2024-11-21T03:26:03,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742310_1486 (size=12001) 2024-11-21T03:26:03,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159623342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159623343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159623345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159623346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159623347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159623458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159623458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159623458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159623467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159623467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159623668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159623668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159623678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159623678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159623687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,702 WARN [regionserver/b69e2a1ee061:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-21T03:26:03,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/93f6fc8e43844297bdfcf85b92396cb2 2024-11-21T03:26:03,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/239d480c8bbd442d8205acefe57b702d is 50, key is test_row_0/C:col10/1732159562130/Put/seqid=0 2024-11-21T03:26:03,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742311_1487 (size=12001) 2024-11-21T03:26:03,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159623978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159623987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159623987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159623988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159623998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/239d480c8bbd442d8205acefe57b702d 2024-11-21T03:26:04,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/a81599aa4f7b4880875222247de618bd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/a81599aa4f7b4880875222247de618bd 2024-11-21T03:26:04,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/a81599aa4f7b4880875222247de618bd, entries=200, sequenceid=51, filesize=14.0 K 2024-11-21T03:26:04,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/93f6fc8e43844297bdfcf85b92396cb2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/93f6fc8e43844297bdfcf85b92396cb2 2024-11-21T03:26:04,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/93f6fc8e43844297bdfcf85b92396cb2, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T03:26:04,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/239d480c8bbd442d8205acefe57b702d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/239d480c8bbd442d8205acefe57b702d 2024-11-21T03:26:04,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/239d480c8bbd442d8205acefe57b702d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T03:26:04,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 215fc242d64dec572e979318d587db2b in 895ms, sequenceid=51, compaction requested=true 2024-11-21T03:26:04,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:04,197 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:04,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:04,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:04,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:04,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:04,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:04,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:26:04,198 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:04,198 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:04,199 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:04,199 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:04,199 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:04,199 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/642116801803413c9fbdd483f86d4d36, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f0d93da9f6745418ecd111fefcfbddd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/a81599aa4f7b4880875222247de618bd] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=37.4 K 2024-11-21T03:26:04,199 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:04,199 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:04,199 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/f22cdff474c1476da7c4985d2e850f4a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c747bf6c55f0476c9ce09e1de1bffd4d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/93f6fc8e43844297bdfcf85b92396cb2] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=35.2 K 2024-11-21T03:26:04,199 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f22cdff474c1476da7c4985d2e850f4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732159560824 2024-11-21T03:26:04,199 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 642116801803413c9fbdd483f86d4d36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732159560824 2024-11-21T03:26:04,200 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f0d93da9f6745418ecd111fefcfbddd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732159560889 2024-11-21T03:26:04,200 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c747bf6c55f0476c9ce09e1de1bffd4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732159560889 2024-11-21T03:26:04,200 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting a81599aa4f7b4880875222247de618bd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159562130 2024-11-21T03:26:04,200 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f6fc8e43844297bdfcf85b92396cb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159562130 2024-11-21T03:26:04,208 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:04,209 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/5d1d3eaaa62e4097b3dbe1c52f01d8f4 is 50, key is test_row_0/B:col10/1732159562130/Put/seqid=0 2024-11-21T03:26:04,210 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:04,210 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/19b18c67f83841088cc76f78cbf0065a is 50, key is test_row_0/A:col10/1732159562130/Put/seqid=0 2024-11-21T03:26:04,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742312_1488 (size=12104) 2024-11-21T03:26:04,243 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/5d1d3eaaa62e4097b3dbe1c52f01d8f4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/5d1d3eaaa62e4097b3dbe1c52f01d8f4 2024-11-21T03:26:04,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742313_1489 (size=12104) 2024-11-21T03:26:04,255 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into 5d1d3eaaa62e4097b3dbe1c52f01d8f4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:04,255 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:04,255 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=13, startTime=1732159564197; duration=0sec 2024-11-21T03:26:04,255 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:04,255 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:04,255 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:04,259 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:04,259 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:04,259 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:04,259 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fcb33bff28a54e9eaf906f22229d1267, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/cd028d5b8fff44a8b0a7a2eb05006001, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/239d480c8bbd442d8205acefe57b702d] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=35.2 K 2024-11-21T03:26:04,259 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcb33bff28a54e9eaf906f22229d1267, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732159560824 2024-11-21T03:26:04,260 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd028d5b8fff44a8b0a7a2eb05006001, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732159560889 2024-11-21T03:26:04,260 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 239d480c8bbd442d8205acefe57b702d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159562130 2024-11-21T03:26:04,281 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#410 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:04,282 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/93fe930013b94cc19417d7fb57f305dd is 50, key is test_row_0/C:col10/1732159562130/Put/seqid=0 2024-11-21T03:26:04,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742314_1490 (size=12104) 2024-11-21T03:26:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:04,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:26:04,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/5dfd9d0b421e4fd5bf0ab1e951b40b7d is 50, key is test_row_0/A:col10/1732159564492/Put/seqid=0 2024-11-21T03:26:04,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742315_1491 (size=14341) 2024-11-21T03:26:04,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159624523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159624525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159624534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159624534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159624536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159624637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159624648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,657 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/19b18c67f83841088cc76f78cbf0065a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/19b18c67f83841088cc76f78cbf0065a 2024-11-21T03:26:04,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159624652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,662 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into 19b18c67f83841088cc76f78cbf0065a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:04,662 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:04,662 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=13, startTime=1732159564196; duration=0sec 2024-11-21T03:26:04,662 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:04,662 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:04,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159624657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159624657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,705 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/93fe930013b94cc19417d7fb57f305dd as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/93fe930013b94cc19417d7fb57f305dd 2024-11-21T03:26:04,719 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into 93fe930013b94cc19417d7fb57f305dd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:04,719 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:04,719 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=13, startTime=1732159564197; duration=0sec 2024-11-21T03:26:04,719 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:04,719 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:04,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159624858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159624867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159624867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159624878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159624880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:04,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/5dfd9d0b421e4fd5bf0ab1e951b40b7d 2024-11-21T03:26:04,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c6d157f2a58f4973bb48b07ce3f2d767 is 50, key is test_row_0/B:col10/1732159564492/Put/seqid=0 2024-11-21T03:26:04,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742316_1492 (size=12001) 2024-11-21T03:26:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:26:05,009 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:05,009 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-21T03:26:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:26:05,011 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:05,012 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:05,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:05,093 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:26:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:26:05,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:26:05,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:05,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159625178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:05,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159625187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:05,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159625187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:05,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159625187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:05,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159625198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:05,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:26:05,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:05,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:26:05,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c6d157f2a58f4973bb48b07ce3f2d767 2024-11-21T03:26:05,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/41cd49f7a54f40ef9252464230511cc4 is 50, key is test_row_0/C:col10/1732159564492/Put/seqid=0 2024-11-21T03:26:05,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742317_1493 (size=12001) 2024-11-21T03:26:05,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:26:05,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:05,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:26:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:26:05,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159625698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:05,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159625698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159625698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:05,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:05,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159625708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159625708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:05,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:05,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:26:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:05,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/41cd49f7a54f40ef9252464230511cc4 2024-11-21T03:26:05,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/5dfd9d0b421e4fd5bf0ab1e951b40b7d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5dfd9d0b421e4fd5bf0ab1e951b40b7d 2024-11-21T03:26:05,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5dfd9d0b421e4fd5bf0ab1e951b40b7d, entries=200, sequenceid=75, filesize=14.0 K 2024-11-21T03:26:05,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c6d157f2a58f4973bb48b07ce3f2d767 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c6d157f2a58f4973bb48b07ce3f2d767 2024-11-21T03:26:05,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c6d157f2a58f4973bb48b07ce3f2d767, entries=150, sequenceid=75, filesize=11.7 K 2024-11-21T03:26:05,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/41cd49f7a54f40ef9252464230511cc4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/41cd49f7a54f40ef9252464230511cc4 2024-11-21T03:26:05,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/41cd49f7a54f40ef9252464230511cc4, entries=150, sequenceid=75, filesize=11.7 K 2024-11-21T03:26:05,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 215fc242d64dec572e979318d587db2b in 1335ms, sequenceid=75, compaction requested=false 2024-11-21T03:26:05,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:05,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:26:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:05,943 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:26:05,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/aedb5b24625b4339982c7eca2b86b2fc is 50, key is test_row_0/A:col10/1732159564532/Put/seqid=0 2024-11-21T03:26:05,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742318_1494 (size=12001) 2024-11-21T03:26:06,013 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/aedb5b24625b4339982c7eca2b86b2fc 2024-11-21T03:26:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c9079b1204ea4fe39a3dfd41ad430ddf is 50, key is test_row_0/B:col10/1732159564532/Put/seqid=0 2024-11-21T03:26:06,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742319_1495 (size=12001) 2024-11-21T03:26:06,104 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c9079b1204ea4fe39a3dfd41ad430ddf 2024-11-21T03:26:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:26:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/3e738b6c532c4bcbb78cc5497ede8147 is 50, key is test_row_0/C:col10/1732159564532/Put/seqid=0 2024-11-21T03:26:06,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742320_1496 (size=12001) 2024-11-21T03:26:06,212 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/3e738b6c532c4bcbb78cc5497ede8147 2024-11-21T03:26:06,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/aedb5b24625b4339982c7eca2b86b2fc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/aedb5b24625b4339982c7eca2b86b2fc 2024-11-21T03:26:06,279 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/aedb5b24625b4339982c7eca2b86b2fc, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T03:26:06,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c9079b1204ea4fe39a3dfd41ad430ddf as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c9079b1204ea4fe39a3dfd41ad430ddf 2024-11-21T03:26:06,343 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c9079b1204ea4fe39a3dfd41ad430ddf, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T03:26:06,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/3e738b6c532c4bcbb78cc5497ede8147 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3e738b6c532c4bcbb78cc5497ede8147 2024-11-21T03:26:06,415 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3e738b6c532c4bcbb78cc5497ede8147, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T03:26:06,424 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 215fc242d64dec572e979318d587db2b in 481ms, sequenceid=90, compaction requested=true 2024-11-21T03:26:06,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:06,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:06,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-21T03:26:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-21T03:26:06,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-21T03:26:06,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-11-21T03:26:06,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.4310 sec 2024-11-21T03:26:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:06,760 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:06,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:26:06,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/83a980bc371647f4b40708f992ebff5b is 50, key is test_row_0/A:col10/1732159566757/Put/seqid=0 2024-11-21T03:26:06,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742321_1497 (size=16681) 2024-11-21T03:26:06,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/83a980bc371647f4b40708f992ebff5b 2024-11-21T03:26:06,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/90810f53c6c84f1b97d1a31e0297f369 is 50, key is test_row_0/B:col10/1732159566757/Put/seqid=0 2024-11-21T03:26:06,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742322_1498 (size=12001) 2024-11-21T03:26:06,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:06,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/90810f53c6c84f1b97d1a31e0297f369 2024-11-21T03:26:06,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159626928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:06,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:06,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159626936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:06,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:06,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159626936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:06,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159626957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159626957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/bddf570f18e14db3b665c45fc933bb09 is 50, key is test_row_0/C:col10/1732159566757/Put/seqid=0 2024-11-21T03:26:07,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159627070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159627071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159627073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742323_1499 (size=12001) 2024-11-21T03:26:07,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159627091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159627091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/bddf570f18e14db3b665c45fc933bb09 2024-11-21T03:26:07,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/83a980bc371647f4b40708f992ebff5b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/83a980bc371647f4b40708f992ebff5b 2024-11-21T03:26:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:26:07,170 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:07,170 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-21T03:26:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:26:07,177 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:07,178 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:07,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:07,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/83a980bc371647f4b40708f992ebff5b, entries=250, sequenceid=103, filesize=16.3 K 2024-11-21T03:26:07,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/90810f53c6c84f1b97d1a31e0297f369 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/90810f53c6c84f1b97d1a31e0297f369 2024-11-21T03:26:07,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/90810f53c6c84f1b97d1a31e0297f369, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T03:26:07,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/bddf570f18e14db3b665c45fc933bb09 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bddf570f18e14db3b665c45fc933bb09 2024-11-21T03:26:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:26:07,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bddf570f18e14db3b665c45fc933bb09, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T03:26:07,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 215fc242d64dec572e979318d587db2b in 532ms, sequenceid=103, compaction requested=true 2024-11-21T03:26:07,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:07,292 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:26:07,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:07,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:07,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:07,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:07,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:07,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:26:07,293 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:26:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:07,320 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:07,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:26:07,325 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:26:07,325 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:07,325 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,325 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/5d1d3eaaa62e4097b3dbe1c52f01d8f4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c6d157f2a58f4973bb48b07ce3f2d767, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c9079b1204ea4fe39a3dfd41ad430ddf, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/90810f53c6c84f1b97d1a31e0297f369] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=47.0 K 2024-11-21T03:26:07,326 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:26:07,326 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:07,326 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,326 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/19b18c67f83841088cc76f78cbf0065a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5dfd9d0b421e4fd5bf0ab1e951b40b7d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/aedb5b24625b4339982c7eca2b86b2fc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/83a980bc371647f4b40708f992ebff5b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=53.8 K 2024-11-21T03:26:07,332 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d1d3eaaa62e4097b3dbe1c52f01d8f4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159562130 2024-11-21T03:26:07,333 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 19b18c67f83841088cc76f78cbf0065a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159562130 2024-11-21T03:26:07,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:26:07,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:07,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,340 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d157f2a58f4973bb48b07ce3f2d767, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732159563343 2024-11-21T03:26:07,341 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dfd9d0b421e4fd5bf0ab1e951b40b7d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732159563343 2024-11-21T03:26:07,345 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9079b1204ea4fe39a3dfd41ad430ddf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732159564531 2024-11-21T03:26:07,345 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting aedb5b24625b4339982c7eca2b86b2fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732159564531 2024-11-21T03:26:07,364 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90810f53c6c84f1b97d1a31e0297f369, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732159566747 2024-11-21T03:26:07,366 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 83a980bc371647f4b40708f992ebff5b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732159566738 2024-11-21T03:26:07,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/afd3c594d4d74f43aa1583958be83e68 is 50, key is test_row_0/A:col10/1732159566926/Put/seqid=0 2024-11-21T03:26:07,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159627369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159627371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159627371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159627378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,407 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:07,407 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/d20e9bbb707545a595218b44ee3759a2 is 50, key is test_row_0/B:col10/1732159566757/Put/seqid=0 2024-11-21T03:26:07,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159627389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,415 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:07,415 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/0588850dda3a48d9a8051808718f52e1 is 50, key is test_row_0/A:col10/1732159566757/Put/seqid=0 2024-11-21T03:26:07,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742324_1500 (size=12001) 2024-11-21T03:26:07,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/afd3c594d4d74f43aa1583958be83e68 2024-11-21T03:26:07,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742325_1501 (size=12241) 2024-11-21T03:26:07,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c61d8555db204d68bf64d3003adc7f36 is 50, key is test_row_0/B:col10/1732159566926/Put/seqid=0 2024-11-21T03:26:07,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742326_1502 (size=12241) 2024-11-21T03:26:07,479 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/d20e9bbb707545a595218b44ee3759a2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d20e9bbb707545a595218b44ee3759a2 2024-11-21T03:26:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:26:07,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:26:07,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:07,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742327_1503 (size=12001) 2024-11-21T03:26:07,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159627498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159627498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159627507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159627508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,533 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into d20e9bbb707545a595218b44ee3759a2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:07,534 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:07,534 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=12, startTime=1732159567292; duration=0sec 2024-11-21T03:26:07,534 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:07,534 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:07,534 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:26:07,535 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:26:07,535 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:07,535 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,535 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/93fe930013b94cc19417d7fb57f305dd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/41cd49f7a54f40ef9252464230511cc4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3e738b6c532c4bcbb78cc5497ede8147, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bddf570f18e14db3b665c45fc933bb09] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=47.0 K 2024-11-21T03:26:07,536 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93fe930013b94cc19417d7fb57f305dd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732159562130 2024-11-21T03:26:07,536 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41cd49f7a54f40ef9252464230511cc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732159563343 2024-11-21T03:26:07,537 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e738b6c532c4bcbb78cc5497ede8147, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732159564531 2024-11-21T03:26:07,537 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting bddf570f18e14db3b665c45fc933bb09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732159566747 2024-11-21T03:26:07,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159627532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,563 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:07,563 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/f0760736e07d46518a0ee0425ea1dc7d is 50, key is test_row_0/C:col10/1732159566757/Put/seqid=0 2024-11-21T03:26:07,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742328_1504 (size=12241) 2024-11-21T03:26:07,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:26:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159627727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159627727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159627738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159627738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159627757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:07,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:07,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:26:07,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:26:07,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:07,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,907 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/0588850dda3a48d9a8051808718f52e1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/0588850dda3a48d9a8051808718f52e1 2024-11-21T03:26:07,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c61d8555db204d68bf64d3003adc7f36 2024-11-21T03:26:07,918 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into 0588850dda3a48d9a8051808718f52e1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:07,918 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:07,918 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=12, startTime=1732159567292; duration=0sec 2024-11-21T03:26:07,918 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:07,918 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:07,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/429a3c64d3f64a3b8d27455c97552007 is 50, key is test_row_0/C:col10/1732159566926/Put/seqid=0 2024-11-21T03:26:07,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:26:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:07,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:07,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742329_1505 (size=12001) 2024-11-21T03:26:07,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/429a3c64d3f64a3b8d27455c97552007 2024-11-21T03:26:08,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/afd3c594d4d74f43aa1583958be83e68 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/afd3c594d4d74f43aa1583958be83e68 2024-11-21T03:26:08,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/afd3c594d4d74f43aa1583958be83e68, entries=150, sequenceid=128, filesize=11.7 K 2024-11-21T03:26:08,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c61d8555db204d68bf64d3003adc7f36 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c61d8555db204d68bf64d3003adc7f36 2024-11-21T03:26:08,026 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/f0760736e07d46518a0ee0425ea1dc7d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/f0760736e07d46518a0ee0425ea1dc7d 2024-11-21T03:26:08,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c61d8555db204d68bf64d3003adc7f36, entries=150, sequenceid=128, filesize=11.7 K 2024-11-21T03:26:08,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/429a3c64d3f64a3b8d27455c97552007 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/429a3c64d3f64a3b8d27455c97552007 2024-11-21T03:26:08,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/429a3c64d3f64a3b8d27455c97552007, entries=150, sequenceid=128, filesize=11.7 K 2024-11-21T03:26:08,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 215fc242d64dec572e979318d587db2b in 718ms, sequenceid=128, compaction requested=false 2024-11-21T03:26:08,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:08,039 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into f0760736e07d46518a0ee0425ea1dc7d(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:08,039 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:08,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:26:08,041 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:08,041 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=12, startTime=1732159567293; duration=0sec 2024-11-21T03:26:08,041 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:08,041 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:08,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/d5a6709f6cf14dbf8e881db67205ca01 is 50, key is test_row_0/A:col10/1732159567371/Put/seqid=0 2024-11-21T03:26:08,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:08,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742330_1506 (size=14541) 2024-11-21T03:26:08,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/d5a6709f6cf14dbf8e881db67205ca01 2024-11-21T03:26:08,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/e19bb53f34504778ab4ed648be06e5ac is 50, key is test_row_0/B:col10/1732159567371/Put/seqid=0 2024-11-21T03:26:08,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742331_1507 (size=12151) 2024-11-21T03:26:08,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/e19bb53f34504778ab4ed648be06e5ac 2024-11-21T03:26:08,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/c535a44594eb468c92595ae714b09304 is 50, key is test_row_0/C:col10/1732159567371/Put/seqid=0 2024-11-21T03:26:08,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742332_1508 (size=12151) 2024-11-21T03:26:08,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/c535a44594eb468c92595ae714b09304 2024-11-21T03:26:08,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:26:08,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:08,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:08,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:08,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:08,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:08,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/d5a6709f6cf14dbf8e881db67205ca01 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/d5a6709f6cf14dbf8e881db67205ca01 2024-11-21T03:26:08,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/d5a6709f6cf14dbf8e881db67205ca01, entries=200, sequenceid=143, filesize=14.2 K 2024-11-21T03:26:08,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/e19bb53f34504778ab4ed648be06e5ac as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e19bb53f34504778ab4ed648be06e5ac 2024-11-21T03:26:08,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e19bb53f34504778ab4ed648be06e5ac, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T03:26:08,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/c535a44594eb468c92595ae714b09304 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/c535a44594eb468c92595ae714b09304 2024-11-21T03:26:08,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159628129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159628130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159628130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159628131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/c535a44594eb468c92595ae714b09304, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T03:26:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159628131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 215fc242d64dec572e979318d587db2b in 108ms, sequenceid=143, compaction requested=true 2024-11-21T03:26:08,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:08,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:08,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:08,148 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:08,148 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:08,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:08,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:08,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:08,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:08,149 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:08,149 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:08,149 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:08,150 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/0588850dda3a48d9a8051808718f52e1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/afd3c594d4d74f43aa1583958be83e68, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/d5a6709f6cf14dbf8e881db67205ca01] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=37.9 K 2024-11-21T03:26:08,150 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:08,151 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:08,151 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:08,151 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d20e9bbb707545a595218b44ee3759a2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c61d8555db204d68bf64d3003adc7f36, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e19bb53f34504778ab4ed648be06e5ac] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=35.5 K 2024-11-21T03:26:08,151 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0588850dda3a48d9a8051808718f52e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732159566747 2024-11-21T03:26:08,152 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d20e9bbb707545a595218b44ee3759a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732159566747 2024-11-21T03:26:08,152 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd3c594d4d74f43aa1583958be83e68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732159566926 2024-11-21T03:26:08,152 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting c61d8555db204d68bf64d3003adc7f36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732159566926 2024-11-21T03:26:08,153 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a6709f6cf14dbf8e881db67205ca01, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732159567328 2024-11-21T03:26:08,153 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e19bb53f34504778ab4ed648be06e5ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732159567369 2024-11-21T03:26:08,187 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:08,187 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/f515c7876e4742c481b5389544b972ba is 50, key is test_row_0/A:col10/1732159567371/Put/seqid=0 2024-11-21T03:26:08,203 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:08,204 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/0be37e921f404e409aaf3243cfe982be is 50, key is test_row_0/B:col10/1732159567371/Put/seqid=0 2024-11-21T03:26:08,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742333_1509 (size=12493) 2024-11-21T03:26:08,261 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/f515c7876e4742c481b5389544b972ba as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/f515c7876e4742c481b5389544b972ba 2024-11-21T03:26:08,276 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into f515c7876e4742c481b5389544b972ba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:08,276 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:08,277 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=13, startTime=1732159568147; duration=0sec 2024-11-21T03:26:08,277 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:08,277 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:08,277 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:08,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742334_1510 (size=12493) 2024-11-21T03:26:08,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:26:08,278 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:08,278 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:08,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:08,278 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:08,279 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/f0760736e07d46518a0ee0425ea1dc7d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/429a3c64d3f64a3b8d27455c97552007, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/c535a44594eb468c92595ae714b09304] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=35.5 K 2024-11-21T03:26:08,279 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:26:08,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:08,280 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0760736e07d46518a0ee0425ea1dc7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732159566747 2024-11-21T03:26:08,280 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 429a3c64d3f64a3b8d27455c97552007, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732159566926 2024-11-21T03:26:08,281 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c535a44594eb468c92595ae714b09304, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732159567369 2024-11-21T03:26:08,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/94093bfa59d94790a8b7b1af0b511c91 is 50, key is test_row_0/A:col10/1732159568126/Put/seqid=0 2024-11-21T03:26:08,291 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:08,291 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/0be37e921f404e409aaf3243cfe982be as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0be37e921f404e409aaf3243cfe982be 2024-11-21T03:26:08,292 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/056c3b2c07b9409b919c3a498973a3c3 is 50, key is test_row_0/C:col10/1732159567371/Put/seqid=0 2024-11-21T03:26:08,300 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into 0be37e921f404e409aaf3243cfe982be(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:08,300 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:08,300 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=13, startTime=1732159568148; duration=0sec 2024-11-21T03:26:08,301 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:08,301 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:26:08,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159628309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742335_1511 (size=14541) 2024-11-21T03:26:08,326 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/94093bfa59d94790a8b7b1af0b511c91 2024-11-21T03:26:08,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159628310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742336_1512 (size=12493) 2024-11-21T03:26:08,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159628314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4f1d4e0e5e744d949fffb5031f7e6fdf is 50, key is test_row_0/B:col10/1732159568126/Put/seqid=0 2024-11-21T03:26:08,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159628317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159628318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,345 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/056c3b2c07b9409b919c3a498973a3c3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/056c3b2c07b9409b919c3a498973a3c3 2024-11-21T03:26:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,352 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into 056c3b2c07b9409b919c3a498973a3c3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:08,352 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:08,352 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=13, startTime=1732159568148; duration=0sec 2024-11-21T03:26:08,353 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:08,353 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:08,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742337_1513 (size=12151) 2024-11-21T03:26:08,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159628427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159628448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159628448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159628458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159628458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159628648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159628668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159628668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159628668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159628680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:08,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:08,788 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4f1d4e0e5e744d949fffb5031f7e6fdf 2024-11-21T03:26:08,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/2a266fdc7bc8481891b804873dd4d5b1 is 50, key is test_row_0/C:col10/1732159568126/Put/seqid=0 2024-11-21T03:26:08,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742338_1514 (size=12151) 2024-11-21T03:26:08,869 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/2a266fdc7bc8481891b804873dd4d5b1 2024-11-21T03:26:08,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/94093bfa59d94790a8b7b1af0b511c91 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/94093bfa59d94790a8b7b1af0b511c91 2024-11-21T03:26:08,883 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/94093bfa59d94790a8b7b1af0b511c91, entries=200, sequenceid=170, filesize=14.2 K 2024-11-21T03:26:08,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4f1d4e0e5e744d949fffb5031f7e6fdf as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4f1d4e0e5e744d949fffb5031f7e6fdf 2024-11-21T03:26:08,895 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4f1d4e0e5e744d949fffb5031f7e6fdf, entries=150, sequenceid=170, filesize=11.9 K 2024-11-21T03:26:08,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/2a266fdc7bc8481891b804873dd4d5b1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/2a266fdc7bc8481891b804873dd4d5b1 2024-11-21T03:26:08,938 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/2a266fdc7bc8481891b804873dd4d5b1, entries=150, sequenceid=170, filesize=11.9 K 2024-11-21T03:26:08,939 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 215fc242d64dec572e979318d587db2b in 660ms, sequenceid=170, compaction requested=false 2024-11-21T03:26:08,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:08,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:08,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-21T03:26:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-21T03:26:08,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-21T03:26:08,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7710 sec 2024-11-21T03:26:08,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.7840 sec 2024-11-21T03:26:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:08,987 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:08,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:26:09,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/5a1dcb3dd82041f5b77422dcc948510e is 50, key is test_row_0/A:col10/1732159568317/Put/seqid=0 2024-11-21T03:26:09,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742339_1515 (size=14541) 2024-11-21T03:26:09,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159629107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159629108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159629109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159629110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159629114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159629230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159629240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159629247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159629247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159629248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:26:09,318 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:09,318 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-21T03:26:09,320 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:26:09,320 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:09,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:26:09,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/5a1dcb3dd82041f5b77422dcc948510e 2024-11-21T03:26:09,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159629457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159629458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159629458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159629458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159629458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T03:26:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:09,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4de7c3f207a1455192f13d332d18f65c is 50, key is test_row_0/B:col10/1732159568317/Put/seqid=0 2024-11-21T03:26:09,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742340_1516 (size=12151) 2024-11-21T03:26:09,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T03:26:09,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:09,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:09,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:09,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:26:09,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T03:26:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:09,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159629779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159629779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159629779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159629779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159629779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:09,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T03:26:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:09,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4de7c3f207a1455192f13d332d18f65c 2024-11-21T03:26:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:09,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/b13cdb792a4640498f065b736e361099 is 50, key is test_row_0/C:col10/1732159568317/Put/seqid=0 2024-11-21T03:26:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:26:10,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742341_1517 (size=12151) 2024-11-21T03:26:10,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/b13cdb792a4640498f065b736e361099 2024-11-21T03:26:10,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/5a1dcb3dd82041f5b77422dcc948510e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5a1dcb3dd82041f5b77422dcc948510e 2024-11-21T03:26:10,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5a1dcb3dd82041f5b77422dcc948510e, entries=200, sequenceid=183, filesize=14.2 K 2024-11-21T03:26:10,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4de7c3f207a1455192f13d332d18f65c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4de7c3f207a1455192f13d332d18f65c 2024-11-21T03:26:10,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4de7c3f207a1455192f13d332d18f65c, entries=150, sequenceid=183, filesize=11.9 K 2024-11-21T03:26:10,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/b13cdb792a4640498f065b736e361099 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/b13cdb792a4640498f065b736e361099 2024-11-21T03:26:10,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/b13cdb792a4640498f065b736e361099, entries=150, sequenceid=183, filesize=11.9 K 2024-11-21T03:26:10,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 215fc242d64dec572e979318d587db2b in 1042ms, sequenceid=183, compaction requested=true 2024-11-21T03:26:10,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:10,029 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:10,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:10,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:10,030 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:10,031 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:10,031 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:10,031 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,031 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/f515c7876e4742c481b5389544b972ba, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/94093bfa59d94790a8b7b1af0b511c91, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5a1dcb3dd82041f5b77422dcc948510e] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=40.6 K 2024-11-21T03:26:10,031 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f515c7876e4742c481b5389544b972ba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732159567369 2024-11-21T03:26:10,031 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:10,032 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:10,032 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,032 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0be37e921f404e409aaf3243cfe982be, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4f1d4e0e5e744d949fffb5031f7e6fdf, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4de7c3f207a1455192f13d332d18f65c] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=35.9 K 2024-11-21T03:26:10,032 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94093bfa59d94790a8b7b1af0b511c91, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732159568126 2024-11-21T03:26:10,032 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0be37e921f404e409aaf3243cfe982be, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732159567369 2024-11-21T03:26:10,032 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f1d4e0e5e744d949fffb5031f7e6fdf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732159568126 2024-11-21T03:26:10,032 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a1dcb3dd82041f5b77422dcc948510e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732159568310 2024-11-21T03:26:10,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:10,033 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4de7c3f207a1455192f13d332d18f65c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732159568310 2024-11-21T03:26:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:10,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:10,050 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:10,050 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/b625de4b4331432ca1f4a015265c9f4d is 50, key is test_row_0/B:col10/1732159568317/Put/seqid=0 2024-11-21T03:26:10,062 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:10,063 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/9cc348d8138b49a79d48b7cf6bdd263a is 50, key is test_row_0/A:col10/1732159568317/Put/seqid=0 2024-11-21T03:26:10,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-21T03:26:10,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,089 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:26:10,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742342_1518 (size=12595) 2024-11-21T03:26:10,103 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/b625de4b4331432ca1f4a015265c9f4d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/b625de4b4331432ca1f4a015265c9f4d 2024-11-21T03:26:10,110 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into b625de4b4331432ca1f4a015265c9f4d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:10,111 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:10,111 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=13, startTime=1732159570030; duration=0sec 2024-11-21T03:26:10,111 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:10,111 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:10,111 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:10,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742343_1519 (size=12595) 2024-11-21T03:26:10,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/6732a41314fa461bbef6e7377a97273c is 50, key is test_row_0/A:col10/1732159569103/Put/seqid=0 2024-11-21T03:26:10,119 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:10,119 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:10,119 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,119 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/056c3b2c07b9409b919c3a498973a3c3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/2a266fdc7bc8481891b804873dd4d5b1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/b13cdb792a4640498f065b736e361099] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=35.9 K 2024-11-21T03:26:10,120 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 056c3b2c07b9409b919c3a498973a3c3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732159567369 2024-11-21T03:26:10,121 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a266fdc7bc8481891b804873dd4d5b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732159568126 2024-11-21T03:26:10,122 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting b13cdb792a4640498f065b736e361099, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732159568310 2024-11-21T03:26:10,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742344_1520 (size=12151) 2024-11-21T03:26:10,125 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/6732a41314fa461bbef6e7377a97273c 2024-11-21T03:26:10,126 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/9cc348d8138b49a79d48b7cf6bdd263a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9cc348d8138b49a79d48b7cf6bdd263a 2024-11-21T03:26:10,134 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into 9cc348d8138b49a79d48b7cf6bdd263a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:10,134 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:10,134 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=13, startTime=1732159570029; duration=0sec 2024-11-21T03:26:10,134 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:10,134 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:10,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/0455dd8ae13f4987a702bdc4bd26b95b is 50, key is test_row_0/B:col10/1732159569103/Put/seqid=0 2024-11-21T03:26:10,167 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:10,167 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/4fe80342ca9345be876d9f7f3e72cf1f is 50, key is test_row_0/C:col10/1732159568317/Put/seqid=0 2024-11-21T03:26:10,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742345_1521 (size=12151) 2024-11-21T03:26:10,215 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/0455dd8ae13f4987a702bdc4bd26b95b 2024-11-21T03:26:10,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/89f8af56b1d34da3a24daf5a35226197 is 50, key is test_row_0/C:col10/1732159569103/Put/seqid=0 2024-11-21T03:26:10,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742346_1522 (size=12595) 2024-11-21T03:26:10,250 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/4fe80342ca9345be876d9f7f3e72cf1f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4fe80342ca9345be876d9f7f3e72cf1f 2024-11-21T03:26:10,258 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into 4fe80342ca9345be876d9f7f3e72cf1f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:10,258 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:10,258 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=13, startTime=1732159570040; duration=0sec 2024-11-21T03:26:10,258 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:10,258 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:10,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742347_1523 (size=12151) 2024-11-21T03:26:10,274 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/89f8af56b1d34da3a24daf5a35226197 2024-11-21T03:26:10,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/6732a41314fa461bbef6e7377a97273c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6732a41314fa461bbef6e7377a97273c 2024-11-21T03:26:10,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:10,314 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6732a41314fa461bbef6e7377a97273c, entries=150, sequenceid=208, filesize=11.9 K 2024-11-21T03:26:10,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/0455dd8ae13f4987a702bdc4bd26b95b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0455dd8ae13f4987a702bdc4bd26b95b 2024-11-21T03:26:10,319 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0455dd8ae13f4987a702bdc4bd26b95b, entries=150, sequenceid=208, filesize=11.9 K 2024-11-21T03:26:10,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/89f8af56b1d34da3a24daf5a35226197 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/89f8af56b1d34da3a24daf5a35226197 2024-11-21T03:26:10,324 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/89f8af56b1d34da3a24daf5a35226197, entries=150, sequenceid=208, filesize=11.9 K 2024-11-21T03:26:10,326 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for 215fc242d64dec572e979318d587db2b in 238ms, sequenceid=208, compaction requested=false 2024-11-21T03:26:10,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:10,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-21T03:26:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-21T03:26:10,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-21T03:26:10,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-11-21T03:26:10,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.0140 sec 2024-11-21T03:26:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:10,341 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:10,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:26:10,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/91312763d71a4e3b8eb69fbd2b459a1c is 50, key is test_row_0/A:col10/1732159570336/Put/seqid=0 2024-11-21T03:26:10,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742348_1524 (size=12151) 2024-11-21T03:26:10,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/91312763d71a4e3b8eb69fbd2b459a1c 2024-11-21T03:26:10,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/84559797c02440ad9373381fd97c638b is 50, key is test_row_0/B:col10/1732159570336/Put/seqid=0 2024-11-21T03:26:10,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159630425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:26:10,458 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:10,459 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-21T03:26:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T03:26:10,462 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:10,463 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:10,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:10,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742349_1525 (size=12151) 2024-11-21T03:26:10,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159630443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159630444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159630451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159630451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T03:26:10,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159630558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159630578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159630578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159630587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159630587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T03:26:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:10,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:10,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T03:26:10,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:10,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T03:26:10,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159630778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159630787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159630798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159630812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159630812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:10,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/84559797c02440ad9373381fd97c638b 2024-11-21T03:26:10,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/bebbe0cb4cfb4471a0b2efac93b73d02 is 50, key is test_row_0/C:col10/1732159570336/Put/seqid=0 2024-11-21T03:26:10,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742350_1526 (size=12151) 2024-11-21T03:26:10,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/bebbe0cb4cfb4471a0b2efac93b73d02 2024-11-21T03:26:10,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/91312763d71a4e3b8eb69fbd2b459a1c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/91312763d71a4e3b8eb69fbd2b459a1c 2024-11-21T03:26:10,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T03:26:10,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:10,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:10,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:10,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/91312763d71a4e3b8eb69fbd2b459a1c, entries=150, sequenceid=223, filesize=11.9 K 2024-11-21T03:26:10,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/84559797c02440ad9373381fd97c638b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/84559797c02440ad9373381fd97c638b 2024-11-21T03:26:10,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/84559797c02440ad9373381fd97c638b, entries=150, sequenceid=223, filesize=11.9 K 2024-11-21T03:26:10,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/bebbe0cb4cfb4471a0b2efac93b73d02 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bebbe0cb4cfb4471a0b2efac93b73d02 2024-11-21T03:26:10,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bebbe0cb4cfb4471a0b2efac93b73d02, entries=150, sequenceid=223, filesize=11.9 K 2024-11-21T03:26:10,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 215fc242d64dec572e979318d587db2b in 597ms, sequenceid=223, compaction requested=true 2024-11-21T03:26:10,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:10,939 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:10,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:10,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:10,939 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:10,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:10,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:10,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:10,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:10,940 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:10,940 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:10,940 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:10,940 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:10,940 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,940 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/b625de4b4331432ca1f4a015265c9f4d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0455dd8ae13f4987a702bdc4bd26b95b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/84559797c02440ad9373381fd97c638b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.0 K 2024-11-21T03:26:10,940 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:10,941 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9cc348d8138b49a79d48b7cf6bdd263a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6732a41314fa461bbef6e7377a97273c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/91312763d71a4e3b8eb69fbd2b459a1c] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.0 K 2024-11-21T03:26:10,942 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cc348d8138b49a79d48b7cf6bdd263a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732159568310 2024-11-21T03:26:10,942 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting b625de4b4331432ca1f4a015265c9f4d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732159568310 2024-11-21T03:26:10,942 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6732a41314fa461bbef6e7377a97273c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732159569103 2024-11-21T03:26:10,942 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0455dd8ae13f4987a702bdc4bd26b95b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732159569103 2024-11-21T03:26:10,942 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91312763d71a4e3b8eb69fbd2b459a1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732159570336 2024-11-21T03:26:10,942 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 84559797c02440ad9373381fd97c638b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732159570336 2024-11-21T03:26:10,970 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:10,971 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/856d3b361282415eaae07d79d3972604 is 50, key is test_row_0/A:col10/1732159570336/Put/seqid=0 2024-11-21T03:26:10,980 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:10,981 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/3bcc98d13f5347738dd535e35b2661b1 is 50, key is test_row_0/B:col10/1732159570336/Put/seqid=0 2024-11-21T03:26:11,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742351_1527 (size=12697) 2024-11-21T03:26:11,014 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/856d3b361282415eaae07d79d3972604 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/856d3b361282415eaae07d79d3972604 2024-11-21T03:26:11,029 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into 856d3b361282415eaae07d79d3972604(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:11,029 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:11,029 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=13, startTime=1732159570939; duration=0sec 2024-11-21T03:26:11,029 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:11,029 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:11,029 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:11,030 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:11,030 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:11,030 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:11,030 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4fe80342ca9345be876d9f7f3e72cf1f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/89f8af56b1d34da3a24daf5a35226197, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bebbe0cb4cfb4471a0b2efac93b73d02] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.0 K 2024-11-21T03:26:11,031 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe80342ca9345be876d9f7f3e72cf1f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732159568310 2024-11-21T03:26:11,031 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f8af56b1d34da3a24daf5a35226197, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732159569103 2024-11-21T03:26:11,031 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting bebbe0cb4cfb4471a0b2efac93b73d02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732159570336 2024-11-21T03:26:11,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742352_1528 (size=12697) 2024-11-21T03:26:11,046 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/3bcc98d13f5347738dd535e35b2661b1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/3bcc98d13f5347738dd535e35b2661b1 2024-11-21T03:26:11,052 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:11,053 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/f9cefee148a94247bc888c77aec46981 is 50, key is test_row_0/C:col10/1732159570336/Put/seqid=0 2024-11-21T03:26:11,061 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into 3bcc98d13f5347738dd535e35b2661b1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:11,061 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:11,061 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=13, startTime=1732159570939; duration=0sec 2024-11-21T03:26:11,061 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:11,061 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:11,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-21T03:26:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:11,085 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:26:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T03:26:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/ce843858b1204f5eab165241ae678964 is 50, key is test_row_0/A:col10/1732159570448/Put/seqid=0 2024-11-21T03:26:11,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742353_1529 (size=12697) 2024-11-21T03:26:11,107 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/f9cefee148a94247bc888c77aec46981 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/f9cefee148a94247bc888c77aec46981 2024-11-21T03:26:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:11,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:11,120 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into f9cefee148a94247bc888c77aec46981(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:11,120 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:11,120 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=13, startTime=1732159570939; duration=0sec 2024-11-21T03:26:11,120 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:11,120 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:11,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742354_1530 (size=12151) 2024-11-21T03:26:11,127 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/ce843858b1204f5eab165241ae678964 2024-11-21T03:26:11,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/d6524873b51341b0bd6e6d72f58054a5 is 50, key is test_row_0/B:col10/1732159570448/Put/seqid=0 2024-11-21T03:26:11,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159631135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159631139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159631139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159631141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159631142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742355_1531 (size=12151) 2024-11-21T03:26:11,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159631257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159631268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159631268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159631268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159631268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159631468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159631477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159631477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159631477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159631477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,568 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/d6524873b51341b0bd6e6d72f58054a5 2024-11-21T03:26:11,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/8ee19f2e35a04ea0b97f937af8557e4b is 50, key is test_row_0/C:col10/1732159570448/Put/seqid=0 2024-11-21T03:26:11,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742356_1532 (size=12151) 2024-11-21T03:26:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T03:26:11,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159631779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159631798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159631798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159631798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159631798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:11,981 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/8ee19f2e35a04ea0b97f937af8557e4b 2024-11-21T03:26:11,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/ce843858b1204f5eab165241ae678964 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/ce843858b1204f5eab165241ae678964 2024-11-21T03:26:11,989 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/ce843858b1204f5eab165241ae678964, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T03:26:11,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/d6524873b51341b0bd6e6d72f58054a5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d6524873b51341b0bd6e6d72f58054a5 2024-11-21T03:26:11,995 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d6524873b51341b0bd6e6d72f58054a5, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T03:26:11,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/8ee19f2e35a04ea0b97f937af8557e4b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/8ee19f2e35a04ea0b97f937af8557e4b 2024-11-21T03:26:12,000 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/8ee19f2e35a04ea0b97f937af8557e4b, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T03:26:12,004 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 215fc242d64dec572e979318d587db2b in 919ms, sequenceid=250, compaction requested=false 2024-11-21T03:26:12,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:12,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:12,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-21T03:26:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-21T03:26:12,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-21T03:26:12,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5430 sec 2024-11-21T03:26:12,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.5510 sec 2024-11-21T03:26:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:12,304 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:12,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:26:12,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/3033db9f7afc44a4bc9ee2405e6a54a4 is 50, key is test_row_0/A:col10/1732159571140/Put/seqid=0 2024-11-21T03:26:12,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742357_1533 (size=14741) 2024-11-21T03:26:12,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/3033db9f7afc44a4bc9ee2405e6a54a4 2024-11-21T03:26:12,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/fd9b2a2fc37b4918a63a4d77861bf8e3 is 50, key is test_row_0/B:col10/1732159571140/Put/seqid=0 2024-11-21T03:26:12,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159632391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742358_1534 (size=12301) 2024-11-21T03:26:12,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/fd9b2a2fc37b4918a63a4d77861bf8e3 2024-11-21T03:26:12,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159632413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/4a426b6d72cc4ab1880a1d4cd3fdf7db is 50, key is test_row_0/C:col10/1732159571140/Put/seqid=0 2024-11-21T03:26:12,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159632423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159632424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159632426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742359_1535 (size=12301) 2024-11-21T03:26:12,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159632528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159632547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159632558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159632558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159632560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T03:26:12,608 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:12,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-21T03:26:12,610 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T03:26:12,611 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:12,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T03:26:12,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159632747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T03:26:12,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:12,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:12,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:12,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:12,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:12,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159632757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159632767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159632768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159632778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:12,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/4a426b6d72cc4ab1880a1d4cd3fdf7db 2024-11-21T03:26:12,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/3033db9f7afc44a4bc9ee2405e6a54a4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3033db9f7afc44a4bc9ee2405e6a54a4 2024-11-21T03:26:12,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3033db9f7afc44a4bc9ee2405e6a54a4, entries=200, sequenceid=263, filesize=14.4 K 2024-11-21T03:26:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/fd9b2a2fc37b4918a63a4d77861bf8e3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/fd9b2a2fc37b4918a63a4d77861bf8e3 2024-11-21T03:26:12,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/fd9b2a2fc37b4918a63a4d77861bf8e3, entries=150, sequenceid=263, filesize=12.0 K 2024-11-21T03:26:12,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/4a426b6d72cc4ab1880a1d4cd3fdf7db as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4a426b6d72cc4ab1880a1d4cd3fdf7db 2024-11-21T03:26:12,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4a426b6d72cc4ab1880a1d4cd3fdf7db, entries=150, sequenceid=263, filesize=12.0 K 2024-11-21T03:26:12,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 215fc242d64dec572e979318d587db2b in 582ms, sequenceid=263, compaction requested=true 2024-11-21T03:26:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:12,886 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:12,887 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:12,887 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:12,887 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:12,887 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:12,887 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:12,887 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:12,887 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:12,888 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/3bcc98d13f5347738dd535e35b2661b1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d6524873b51341b0bd6e6d72f58054a5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/fd9b2a2fc37b4918a63a4d77861bf8e3] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.3 K 2024-11-21T03:26:12,888 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/856d3b361282415eaae07d79d3972604, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/ce843858b1204f5eab165241ae678964, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3033db9f7afc44a4bc9ee2405e6a54a4] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=38.7 K 2024-11-21T03:26:12,888 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bcc98d13f5347738dd535e35b2661b1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732159570336 2024-11-21T03:26:12,888 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 856d3b361282415eaae07d79d3972604, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732159570336 2024-11-21T03:26:12,888 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce843858b1204f5eab165241ae678964, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732159570414 2024-11-21T03:26:12,888 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d6524873b51341b0bd6e6d72f58054a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732159570414 2024-11-21T03:26:12,888 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3033db9f7afc44a4bc9ee2405e6a54a4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159571136 2024-11-21T03:26:12,888 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting fd9b2a2fc37b4918a63a4d77861bf8e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159571136 2024-11-21T03:26:12,893 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:12,893 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/e314454badbe468e8113cf11da3f2748 is 50, key is test_row_0/B:col10/1732159571140/Put/seqid=0 2024-11-21T03:26:12,893 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:12,894 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/262f0829c422440b8e0cf943202ed0e4 is 50, key is test_row_0/A:col10/1732159571140/Put/seqid=0 2024-11-21T03:26:12,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742360_1536 (size=12949) 2024-11-21T03:26:12,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742361_1537 (size=12949) 2024-11-21T03:26:12,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-21T03:26:12,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:12,917 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:26:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/6a0577f37b7d4fa39cc53ee8ec111c51 is 50, key is test_row_0/A:col10/1732159572413/Put/seqid=0 2024-11-21T03:26:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T03:26:12,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742362_1538 (size=12301) 2024-11-21T03:26:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:13,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:13,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159633088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159633089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159633091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159633097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159633100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159633207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159633207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159633207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159633207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T03:26:13,309 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/e314454badbe468e8113cf11da3f2748 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e314454badbe468e8113cf11da3f2748 2024-11-21T03:26:13,318 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into e314454badbe468e8113cf11da3f2748(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:13,318 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:13,318 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=13, startTime=1732159572886; duration=0sec 2024-11-21T03:26:13,318 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:13,318 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:13,319 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:13,322 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:13,322 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:13,322 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:13,322 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/f9cefee148a94247bc888c77aec46981, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/8ee19f2e35a04ea0b97f937af8557e4b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4a426b6d72cc4ab1880a1d4cd3fdf7db] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.3 K 2024-11-21T03:26:13,323 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting f9cefee148a94247bc888c77aec46981, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732159570336 2024-11-21T03:26:13,323 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ee19f2e35a04ea0b97f937af8557e4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732159570414 2024-11-21T03:26:13,323 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/262f0829c422440b8e0cf943202ed0e4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/262f0829c422440b8e0cf943202ed0e4 2024-11-21T03:26:13,324 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a426b6d72cc4ab1880a1d4cd3fdf7db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159571136 2024-11-21T03:26:13,328 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into 262f0829c422440b8e0cf943202ed0e4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:13,329 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:13,329 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=13, startTime=1732159572886; duration=0sec 2024-11-21T03:26:13,329 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/6a0577f37b7d4fa39cc53ee8ec111c51 2024-11-21T03:26:13,329 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:13,329 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:13,335 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:13,335 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/55608e4e18484acaa778ea3e35ac6784 is 50, key is test_row_0/C:col10/1732159571140/Put/seqid=0 2024-11-21T03:26:13,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/8cdbb89190b442c3b5c0a3b289efb913 is 50, key is test_row_0/B:col10/1732159572413/Put/seqid=0 2024-11-21T03:26:13,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742363_1539 (size=12949) 2024-11-21T03:26:13,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742364_1540 (size=12301) 2024-11-21T03:26:13,360 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/8cdbb89190b442c3b5c0a3b289efb913 2024-11-21T03:26:13,365 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/55608e4e18484acaa778ea3e35ac6784 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/55608e4e18484acaa778ea3e35ac6784 2024-11-21T03:26:13,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/e9d2b70bc0434a7aaf6fee0ad1b5af03 is 50, key is test_row_0/C:col10/1732159572413/Put/seqid=0 2024-11-21T03:26:13,370 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into 55608e4e18484acaa778ea3e35ac6784(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:13,371 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:13,371 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=13, startTime=1732159572886; duration=0sec 2024-11-21T03:26:13,371 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:13,371 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:13,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742365_1541 (size=12301) 2024-11-21T03:26:13,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159633418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159633418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159633418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159633418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159633608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159633738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159633738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159633739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159633739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:13,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T03:26:13,782 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/e9d2b70bc0434a7aaf6fee0ad1b5af03 2024-11-21T03:26:13,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/6a0577f37b7d4fa39cc53ee8ec111c51 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6a0577f37b7d4fa39cc53ee8ec111c51 2024-11-21T03:26:13,814 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6a0577f37b7d4fa39cc53ee8ec111c51, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T03:26:13,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/8cdbb89190b442c3b5c0a3b289efb913 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8cdbb89190b442c3b5c0a3b289efb913 2024-11-21T03:26:13,820 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8cdbb89190b442c3b5c0a3b289efb913, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T03:26:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/e9d2b70bc0434a7aaf6fee0ad1b5af03 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/e9d2b70bc0434a7aaf6fee0ad1b5af03 2024-11-21T03:26:13,825 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/e9d2b70bc0434a7aaf6fee0ad1b5af03, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T03:26:13,826 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 215fc242d64dec572e979318d587db2b in 909ms, sequenceid=288, compaction requested=false 2024-11-21T03:26:13,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-21T03:26:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-21T03:26:13,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-21T03:26:13,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2170 sec 2024-11-21T03:26:13,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.2240 sec 2024-11-21T03:26:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:14,253 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:14,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:26:14,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/4eb7b7863a354896b44616d12aacee81 is 50, key is test_row_0/A:col10/1732159573088/Put/seqid=0 2024-11-21T03:26:14,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742366_1542 (size=17181) 2024-11-21T03:26:14,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/4eb7b7863a354896b44616d12aacee81 2024-11-21T03:26:14,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/6057852be6dd4bcab9875d1e8e63da18 is 50, key is test_row_0/B:col10/1732159573088/Put/seqid=0 2024-11-21T03:26:14,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159634326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159634331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159634333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159634335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742367_1543 (size=12301) 2024-11-21T03:26:14,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/6057852be6dd4bcab9875d1e8e63da18 2024-11-21T03:26:14,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/4dfa975b225b4c14b2a1d46e207cad2a is 50, key is test_row_0/C:col10/1732159573088/Put/seqid=0 2024-11-21T03:26:14,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742368_1544 (size=12301) 2024-11-21T03:26:14,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159634448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159634458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159634458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159634460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159634628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,639 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in 215fc242d64dec572e979318d587db2b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:26:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159634668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159634679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159634679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159634687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T03:26:14,768 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:14,769 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-21T03:26:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T03:26:14,777 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:14,779 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:14,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:14,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/4dfa975b225b4c14b2a1d46e207cad2a 2024-11-21T03:26:14,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/4eb7b7863a354896b44616d12aacee81 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4eb7b7863a354896b44616d12aacee81 2024-11-21T03:26:14,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4eb7b7863a354896b44616d12aacee81, entries=250, sequenceid=304, filesize=16.8 K 2024-11-21T03:26:14,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/6057852be6dd4bcab9875d1e8e63da18 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/6057852be6dd4bcab9875d1e8e63da18 2024-11-21T03:26:14,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/6057852be6dd4bcab9875d1e8e63da18, entries=150, sequenceid=304, filesize=12.0 K 2024-11-21T03:26:14,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/4dfa975b225b4c14b2a1d46e207cad2a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4dfa975b225b4c14b2a1d46e207cad2a 2024-11-21T03:26:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T03:26:14,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4dfa975b225b4c14b2a1d46e207cad2a, entries=150, sequenceid=304, filesize=12.0 K 2024-11-21T03:26:14,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 215fc242d64dec572e979318d587db2b in 626ms, sequenceid=304, compaction requested=true 2024-11-21T03:26:14,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:14,879 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:14,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:14,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:14,880 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:14,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:14,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:14,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:14,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:14,882 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:14,882 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:14,882 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:14,883 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/262f0829c422440b8e0cf943202ed0e4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6a0577f37b7d4fa39cc53ee8ec111c51, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4eb7b7863a354896b44616d12aacee81] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=41.4 K 2024-11-21T03:26:14,883 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:14,883 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:14,883 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:14,883 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e314454badbe468e8113cf11da3f2748, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8cdbb89190b442c3b5c0a3b289efb913, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/6057852be6dd4bcab9875d1e8e63da18] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.7 K 2024-11-21T03:26:14,883 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 262f0829c422440b8e0cf943202ed0e4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159571136 2024-11-21T03:26:14,883 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e314454badbe468e8113cf11da3f2748, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159571136 2024-11-21T03:26:14,884 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a0577f37b7d4fa39cc53ee8ec111c51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732159572346 2024-11-21T03:26:14,884 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cdbb89190b442c3b5c0a3b289efb913, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732159572346 2024-11-21T03:26:14,884 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eb7b7863a354896b44616d12aacee81, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159573088 2024-11-21T03:26:14,884 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6057852be6dd4bcab9875d1e8e63da18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159573088 2024-11-21T03:26:14,906 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:14,906 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/b5d1de263efb4eb8ae8c528ddc1d215e is 50, key is test_row_0/B:col10/1732159573088/Put/seqid=0 2024-11-21T03:26:14,910 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:14,911 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/e370bcc5ec414abdaf2f0794a26c0ea0 is 50, key is test_row_0/A:col10/1732159573088/Put/seqid=0 2024-11-21T03:26:14,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742369_1545 (size=13051) 2024-11-21T03:26:14,927 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/b5d1de263efb4eb8ae8c528ddc1d215e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/b5d1de263efb4eb8ae8c528ddc1d215e 2024-11-21T03:26:14,932 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into b5d1de263efb4eb8ae8c528ddc1d215e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:14,932 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:14,932 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=13, startTime=1732159574880; duration=0sec 2024-11-21T03:26:14,932 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:14,932 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:14,932 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:14,933 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:14,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742370_1546 (size=13051) 2024-11-21T03:26:14,933 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:14,933 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:14,933 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/55608e4e18484acaa778ea3e35ac6784, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/e9d2b70bc0434a7aaf6fee0ad1b5af03, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4dfa975b225b4c14b2a1d46e207cad2a] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.7 K 2024-11-21T03:26:14,933 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 55608e4e18484acaa778ea3e35ac6784, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732159571136 2024-11-21T03:26:14,934 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d2b70bc0434a7aaf6fee0ad1b5af03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732159572346 2024-11-21T03:26:14,934 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dfa975b225b4c14b2a1d46e207cad2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159573088 2024-11-21T03:26:14,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T03:26:14,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:14,940 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:26:14,941 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:14,942 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/9a9cc1728b7d406398ffefd68c76e491 is 50, key is test_row_0/C:col10/1732159573088/Put/seqid=0 2024-11-21T03:26:14,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742371_1547 (size=13051) 2024-11-21T03:26:14,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/9e42a957026d4ec58624934b316e03aa is 50, key is test_row_0/A:col10/1732159574327/Put/seqid=0 2024-11-21T03:26:14,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:15,007 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/9a9cc1728b7d406398ffefd68c76e491 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/9a9cc1728b7d406398ffefd68c76e491 2024-11-21T03:26:15,017 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into 9a9cc1728b7d406398ffefd68c76e491(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:15,017 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:15,017 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=13, startTime=1732159574881; duration=0sec 2024-11-21T03:26:15,017 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:15,017 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:15,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742372_1548 (size=12301) 2024-11-21T03:26:15,026 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/9e42a957026d4ec58624934b316e03aa 2024-11-21T03:26:15,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c5ef37c7771b479aae0c312a4020e421 is 50, key is test_row_0/B:col10/1732159574327/Put/seqid=0 2024-11-21T03:26:15,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159635023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159635024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742373_1549 (size=12301) 2024-11-21T03:26:15,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159635038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159635038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,054 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c5ef37c7771b479aae0c312a4020e421 2024-11-21T03:26:15,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/99ef0377c98641a981b4a770fea35785 is 50, key is test_row_0/C:col10/1732159574327/Put/seqid=0 2024-11-21T03:26:15,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742374_1550 (size=12301) 2024-11-21T03:26:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T03:26:15,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159635148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159635149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159635161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159635161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,338 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/e370bcc5ec414abdaf2f0794a26c0ea0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/e370bcc5ec414abdaf2f0794a26c0ea0 2024-11-21T03:26:15,341 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into e370bcc5ec414abdaf2f0794a26c0ea0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:15,341 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:15,341 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=13, startTime=1732159574879; duration=0sec 2024-11-21T03:26:15,341 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:15,341 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:15,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159635368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159635368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159635378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159635378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T03:26:15,477 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/99ef0377c98641a981b4a770fea35785 2024-11-21T03:26:15,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/9e42a957026d4ec58624934b316e03aa as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9e42a957026d4ec58624934b316e03aa 2024-11-21T03:26:15,494 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9e42a957026d4ec58624934b316e03aa, entries=150, sequenceid=328, filesize=12.0 K 2024-11-21T03:26:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/c5ef37c7771b479aae0c312a4020e421 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c5ef37c7771b479aae0c312a4020e421 2024-11-21T03:26:15,540 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c5ef37c7771b479aae0c312a4020e421, entries=150, sequenceid=328, filesize=12.0 K 2024-11-21T03:26:15,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/99ef0377c98641a981b4a770fea35785 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/99ef0377c98641a981b4a770fea35785 2024-11-21T03:26:15,544 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/99ef0377c98641a981b4a770fea35785, entries=150, sequenceid=328, filesize=12.0 K 2024-11-21T03:26:15,559 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 215fc242d64dec572e979318d587db2b in 619ms, sequenceid=328, compaction requested=false 2024-11-21T03:26:15,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:15,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:15,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-21T03:26:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-21T03:26:15,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-21T03:26:15,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 785 msec 2024-11-21T03:26:15,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 800 msec 2024-11-21T03:26:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:15,682 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:15,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:26:15,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/4e9af5059ebe46578eed7e87a4da8478 is 50, key is test_row_0/A:col10/1732159575681/Put/seqid=0 2024-11-21T03:26:15,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742375_1551 (size=14741) 2024-11-21T03:26:15,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/4e9af5059ebe46578eed7e87a4da8478 2024-11-21T03:26:15,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/51029712c89642a987f9d6c164da8665 is 50, key is test_row_0/B:col10/1732159575681/Put/seqid=0 2024-11-21T03:26:15,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742376_1552 (size=12301) 2024-11-21T03:26:15,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159635761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159635762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159635762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159635764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159635868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159635877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159635877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159635877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T03:26:15,908 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:15,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-21T03:26:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:26:15,910 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:15,910 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:15,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:26:16,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T03:26:16,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:16,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:16,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:16,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:16,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:16,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159636079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159636088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159636088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159636088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/51029712c89642a987f9d6c164da8665 2024-11-21T03:26:16,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/fe26d1d83298489289340a8df6ef2e26 is 50, key is test_row_0/C:col10/1732159575681/Put/seqid=0 2024-11-21T03:26:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742377_1553 (size=12301) 2024-11-21T03:26:16,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/fe26d1d83298489289340a8df6ef2e26 2024-11-21T03:26:16,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/4e9af5059ebe46578eed7e87a4da8478 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4e9af5059ebe46578eed7e87a4da8478 2024-11-21T03:26:16,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4e9af5059ebe46578eed7e87a4da8478, entries=200, sequenceid=344, filesize=14.4 K 2024-11-21T03:26:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/51029712c89642a987f9d6c164da8665 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/51029712c89642a987f9d6c164da8665 2024-11-21T03:26:16,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T03:26:16,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:16,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:16,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:16,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:16,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:26:16,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/51029712c89642a987f9d6c164da8665, entries=150, sequenceid=344, filesize=12.0 K 2024-11-21T03:26:16,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/fe26d1d83298489289340a8df6ef2e26 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fe26d1d83298489289340a8df6ef2e26 2024-11-21T03:26:16,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fe26d1d83298489289340a8df6ef2e26, entries=150, sequenceid=344, filesize=12.0 K 2024-11-21T03:26:16,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 215fc242d64dec572e979318d587db2b in 620ms, sequenceid=344, compaction requested=true 2024-11-21T03:26:16,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:16,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:16,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:16,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:16,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:26:16,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:16,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T03:26:16,303 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:16,303 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:16,304 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:16,304 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:16,304 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:16,304 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/e370bcc5ec414abdaf2f0794a26c0ea0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9e42a957026d4ec58624934b316e03aa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4e9af5059ebe46578eed7e87a4da8478] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=39.2 K 2024-11-21T03:26:16,305 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:16,305 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting e370bcc5ec414abdaf2f0794a26c0ea0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159573088 2024-11-21T03:26:16,305 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:16,305 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:16,305 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/9a9cc1728b7d406398ffefd68c76e491, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/99ef0377c98641a981b4a770fea35785, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fe26d1d83298489289340a8df6ef2e26] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.8 K 2024-11-21T03:26:16,306 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e42a957026d4ec58624934b316e03aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732159574327 2024-11-21T03:26:16,306 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a9cc1728b7d406398ffefd68c76e491, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159573088 2024-11-21T03:26:16,306 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9af5059ebe46578eed7e87a4da8478, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732159575008 2024-11-21T03:26:16,307 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ef0377c98641a981b4a770fea35785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732159574327 2024-11-21T03:26:16,307 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting fe26d1d83298489289340a8df6ef2e26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732159575035 2024-11-21T03:26:16,338 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:16,339 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/628e2612b04548859ca0e9028eb69788 is 50, key is test_row_0/C:col10/1732159575681/Put/seqid=0 2024-11-21T03:26:16,342 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:16,343 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/9a9f49a1ffa04c3897d04e22a49ddee5 is 50, key is test_row_0/A:col10/1732159575681/Put/seqid=0 2024-11-21T03:26:16,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T03:26:16,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:16,373 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T03:26:16,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742378_1554 (size=13153) 2024-11-21T03:26:16,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/b31848967af94722a926d6c4a9d477d5 is 50, key is test_row_0/A:col10/1732159575761/Put/seqid=0 2024-11-21T03:26:16,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742379_1555 (size=13153) 2024-11-21T03:26:16,400 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/628e2612b04548859ca0e9028eb69788 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/628e2612b04548859ca0e9028eb69788 2024-11-21T03:26:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:16,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:16,417 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/9a9f49a1ffa04c3897d04e22a49ddee5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9a9f49a1ffa04c3897d04e22a49ddee5 2024-11-21T03:26:16,424 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into 9a9f49a1ffa04c3897d04e22a49ddee5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:16,424 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into 628e2612b04548859ca0e9028eb69788(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:16,424 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:16,424 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:16,424 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=13, startTime=1732159576302; duration=0sec 2024-11-21T03:26:16,424 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=13, startTime=1732159576303; duration=0sec 2024-11-21T03:26:16,424 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:16,424 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:16,424 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:16,425 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:16,425 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:16,425 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:16,425 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:16,426 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:16,426 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/b5d1de263efb4eb8ae8c528ddc1d215e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c5ef37c7771b479aae0c312a4020e421, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/51029712c89642a987f9d6c164da8665] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.8 K 2024-11-21T03:26:16,426 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5d1de263efb4eb8ae8c528ddc1d215e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159573088 2024-11-21T03:26:16,426 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ef37c7771b479aae0c312a4020e421, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732159574327 2024-11-21T03:26:16,426 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51029712c89642a987f9d6c164da8665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732159575035 2024-11-21T03:26:16,439 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:16,439 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/0d819a74932f4928b9da8ef2c10e8641 is 50, key is test_row_0/B:col10/1732159575681/Put/seqid=0 2024-11-21T03:26:16,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742380_1556 (size=12301) 2024-11-21T03:26:16,445 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/b31848967af94722a926d6c4a9d477d5 2024-11-21T03:26:16,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4067427061674aac9cd9e8d204bbc233 is 50, key is test_row_0/B:col10/1732159575761/Put/seqid=0 2024-11-21T03:26:16,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742381_1557 (size=13153) 2024-11-21T03:26:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159636469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159636478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742382_1558 (size=12301) 2024-11-21T03:26:16,502 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4067427061674aac9cd9e8d204bbc233 2024-11-21T03:26:16,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159636486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159636487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/96044d9b697f4ccba7c4be3c4d9762b1 is 50, key is test_row_0/C:col10/1732159575761/Put/seqid=0 2024-11-21T03:26:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:26:16,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742383_1559 (size=12301) 2024-11-21T03:26:16,580 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/96044d9b697f4ccba7c4be3c4d9762b1 2024-11-21T03:26:16,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/b31848967af94722a926d6c4a9d477d5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/b31848967af94722a926d6c4a9d477d5 2024-11-21T03:26:16,604 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/b31848967af94722a926d6c4a9d477d5, entries=150, sequenceid=366, filesize=12.0 K 2024-11-21T03:26:16,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4067427061674aac9cd9e8d204bbc233 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4067427061674aac9cd9e8d204bbc233 2024-11-21T03:26:16,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159636598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,625 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4067427061674aac9cd9e8d204bbc233, entries=150, sequenceid=366, filesize=12.0 K 2024-11-21T03:26:16,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/96044d9b697f4ccba7c4be3c4d9762b1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/96044d9b697f4ccba7c4be3c4d9762b1 2024-11-21T03:26:16,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159636617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159636619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159636619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,650 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/96044d9b697f4ccba7c4be3c4d9762b1, entries=150, sequenceid=366, filesize=12.0 K 2024-11-21T03:26:16,651 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 215fc242d64dec572e979318d587db2b in 277ms, sequenceid=366, compaction requested=false 2024-11-21T03:26:16,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:16,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:16,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-21T03:26:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-21T03:26:16,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-21T03:26:16,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 742 msec 2024-11-21T03:26:16,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 747 msec 2024-11-21T03:26:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:16,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T03:26:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:16,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/766ef235739c4d5db487a6daef68334e is 50, key is test_row_0/A:col10/1732159576445/Put/seqid=0 2024-11-21T03:26:16,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742384_1560 (size=14741) 2024-11-21T03:26:16,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159636850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159636852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159636852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159636856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159636858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,898 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/0d819a74932f4928b9da8ef2c10e8641 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0d819a74932f4928b9da8ef2c10e8641 2024-11-21T03:26:16,913 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into 0d819a74932f4928b9da8ef2c10e8641(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:16,913 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:16,913 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=13, startTime=1732159576303; duration=0sec 2024-11-21T03:26:16,913 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:16,913 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:16,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159636969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159636972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:16,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159636988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159636988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:26:17,048 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:17,049 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-21T03:26:17,054 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:17,056 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:17,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:26:17,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/766ef235739c4d5db487a6daef68334e 2024-11-21T03:26:17,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/83954f56af1440a996a01b3415f3fffe is 50, key is test_row_0/B:col10/1732159576445/Put/seqid=0 2024-11-21T03:26:17,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742385_1561 (size=12301) 2024-11-21T03:26:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:26:17,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159637172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159637189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159637192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T03:26:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:17,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159637210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159637220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T03:26:17,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:26:17,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:17,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159637507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159637518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T03:26:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159637528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:17,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159637539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/83954f56af1440a996a01b3415f3fffe 2024-11-21T03:26:17,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/0186fc2388af4c0ca83672ac912994c0 is 50, key is test_row_0/C:col10/1732159576445/Put/seqid=0 2024-11-21T03:26:17,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742386_1562 (size=12301) 2024-11-21T03:26:17,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/0186fc2388af4c0ca83672ac912994c0 2024-11-21T03:26:17,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/766ef235739c4d5db487a6daef68334e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/766ef235739c4d5db487a6daef68334e 2024-11-21T03:26:17,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/766ef235739c4d5db487a6daef68334e, entries=200, sequenceid=383, filesize=14.4 K 2024-11-21T03:26:17,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/83954f56af1440a996a01b3415f3fffe as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/83954f56af1440a996a01b3415f3fffe 2024-11-21T03:26:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:26:17,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T03:26:17,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:17,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/83954f56af1440a996a01b3415f3fffe, entries=150, sequenceid=383, filesize=12.0 K 2024-11-21T03:26:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:17,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/0186fc2388af4c0ca83672ac912994c0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/0186fc2388af4c0ca83672ac912994c0 2024-11-21T03:26:17,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159637700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:17,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/0186fc2388af4c0ca83672ac912994c0, entries=150, sequenceid=383, filesize=12.0 K 2024-11-21T03:26:17,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 215fc242d64dec572e979318d587db2b in 1080ms, sequenceid=383, compaction requested=true 2024-11-21T03:26:17,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:17,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:17,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:17,736 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:17,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:17,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:17,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:17,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:26:17,740 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:17,748 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:17,748 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:17,749 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,749 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9a9f49a1ffa04c3897d04e22a49ddee5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/b31848967af94722a926d6c4a9d477d5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/766ef235739c4d5db487a6daef68334e] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=39.3 K 2024-11-21T03:26:17,757 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a9f49a1ffa04c3897d04e22a49ddee5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732159575035 2024-11-21T03:26:17,762 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting b31848967af94722a926d6c4a9d477d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732159575760 2024-11-21T03:26:17,764 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:17,764 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:17,764 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,764 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0d819a74932f4928b9da8ef2c10e8641, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4067427061674aac9cd9e8d204bbc233, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/83954f56af1440a996a01b3415f3fffe] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.9 K 2024-11-21T03:26:17,772 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 766ef235739c4d5db487a6daef68334e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732159576445 2024-11-21T03:26:17,773 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d819a74932f4928b9da8ef2c10e8641, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732159575035 2024-11-21T03:26:17,777 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4067427061674aac9cd9e8d204bbc233, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732159575760 2024-11-21T03:26:17,780 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83954f56af1440a996a01b3415f3fffe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732159576445 2024-11-21T03:26:17,798 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:17,799 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/1f39c5a203924d9cacef6c11f48b7e2c is 50, key is test_row_0/A:col10/1732159576445/Put/seqid=0 2024-11-21T03:26:17,812 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:17,812 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/dc866f60c1d842048a7bac66bf64f9c8 is 50, key is test_row_0/B:col10/1732159576445/Put/seqid=0 2024-11-21T03:26:17,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T03:26:17,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,852 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T03:26:17,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742388_1564 (size=13255) 2024-11-21T03:26:17,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742387_1563 (size=13255) 2024-11-21T03:26:17,891 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/1f39c5a203924d9cacef6c11f48b7e2c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f39c5a203924d9cacef6c11f48b7e2c 2024-11-21T03:26:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/38c15e05651d45928388369ebfbcbbaa is 50, key is test_row_0/A:col10/1732159576838/Put/seqid=0 2024-11-21T03:26:17,898 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into 1f39c5a203924d9cacef6c11f48b7e2c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:17,898 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:17,899 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=13, startTime=1732159577736; duration=0sec 2024-11-21T03:26:17,899 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:17,899 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:17,899 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:17,905 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:17,905 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:17,905 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:17,905 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/628e2612b04548859ca0e9028eb69788, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/96044d9b697f4ccba7c4be3c4d9762b1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/0186fc2388af4c0ca83672ac912994c0] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=36.9 K 2024-11-21T03:26:17,906 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 628e2612b04548859ca0e9028eb69788, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732159575035 2024-11-21T03:26:17,906 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 96044d9b697f4ccba7c4be3c4d9762b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732159575760 2024-11-21T03:26:17,907 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0186fc2388af4c0ca83672ac912994c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732159576445 2024-11-21T03:26:17,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742389_1565 (size=12301) 2024-11-21T03:26:17,917 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/38c15e05651d45928388369ebfbcbbaa 2024-11-21T03:26:17,934 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:17,935 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/a661f59a81f54ac4ae4099ddd895b577 is 50, key is test_row_0/C:col10/1732159576445/Put/seqid=0 2024-11-21T03:26:17,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/1ce2202fe3ef44e5b38a2ac65d2fc85b is 50, key is test_row_0/B:col10/1732159576838/Put/seqid=0 2024-11-21T03:26:17,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742390_1566 (size=13255) 2024-11-21T03:26:17,999 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/a661f59a81f54ac4ae4099ddd895b577 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a661f59a81f54ac4ae4099ddd895b577 2024-11-21T03:26:18,006 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into a661f59a81f54ac4ae4099ddd895b577(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:18,006 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:18,006 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=13, startTime=1732159577737; duration=0sec 2024-11-21T03:26:18,006 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:18,006 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:18,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742391_1567 (size=12301) 2024-11-21T03:26:18,020 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/1ce2202fe3ef44e5b38a2ac65d2fc85b 2024-11-21T03:26:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/a1f70a947a4a404e980e62f2e202de5b is 50, key is test_row_0/C:col10/1732159576838/Put/seqid=0 2024-11-21T03:26:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:18,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:18,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742392_1568 (size=12301) 2024-11-21T03:26:18,093 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/a1f70a947a4a404e980e62f2e202de5b 2024-11-21T03:26:18,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159638086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/38c15e05651d45928388369ebfbcbbaa as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/38c15e05651d45928388369ebfbcbbaa 2024-11-21T03:26:18,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159638088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,109 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/38c15e05651d45928388369ebfbcbbaa, entries=150, sequenceid=405, filesize=12.0 K 2024-11-21T03:26:18,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/1ce2202fe3ef44e5b38a2ac65d2fc85b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/1ce2202fe3ef44e5b38a2ac65d2fc85b 2024-11-21T03:26:18,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159638093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159638093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,116 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/1ce2202fe3ef44e5b38a2ac65d2fc85b, entries=150, sequenceid=405, filesize=12.0 K 2024-11-21T03:26:18,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/a1f70a947a4a404e980e62f2e202de5b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a1f70a947a4a404e980e62f2e202de5b 2024-11-21T03:26:18,132 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a1f70a947a4a404e980e62f2e202de5b, entries=150, sequenceid=405, filesize=12.0 K 2024-11-21T03:26:18,136 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 215fc242d64dec572e979318d587db2b in 284ms, sequenceid=405, compaction requested=false 2024-11-21T03:26:18,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:18,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-21T03:26:18,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-21T03:26:18,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-21T03:26:18,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0820 sec 2024-11-21T03:26:18,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0920 sec 2024-11-21T03:26:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:26:18,198 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:18,201 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-21T03:26:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:26:18,203 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:18,204 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:18,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:18,221 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:18,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T03:26:18,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/3d0d7f2b42a8465d8c75b2ab167d6989 is 50, key is test_row_0/A:col10/1732159578085/Put/seqid=0 2024-11-21T03:26:18,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742393_1569 (size=14741) 2024-11-21T03:26:18,280 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/dc866f60c1d842048a7bac66bf64f9c8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/dc866f60c1d842048a7bac66bf64f9c8 2024-11-21T03:26:18,285 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into dc866f60c1d842048a7bac66bf64f9c8(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:18,285 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:18,285 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=13, startTime=1732159577736; duration=0sec 2024-11-21T03:26:18,285 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:18,285 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:26:18,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159638320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:26:18,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:18,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159638349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159638358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159638359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159638472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159638478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159638488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:26:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:26:18,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159638499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:18,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:26:18,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:18,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/3d0d7f2b42a8465d8c75b2ab167d6989 2024-11-21T03:26:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159638688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159638697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4551035bb2c14279bf77cd60c607fc74 is 50, key is test_row_0/B:col10/1732159578085/Put/seqid=0 2024-11-21T03:26:18,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159638718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159638730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742394_1570 (size=12301) 2024-11-21T03:26:18,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159638732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:18,747 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in 215fc242d64dec572e979318d587db2b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:26:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4551035bb2c14279bf77cd60c607fc74 2024-11-21T03:26:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:18,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/566cd56a45b64bd3a69a5c99cfa2cd3f is 50, key is test_row_0/C:col10/1732159578085/Put/seqid=0 2024-11-21T03:26:18,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742395_1571 (size=12301) 2024-11-21T03:26:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:26:18,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:26:18,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:18,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:26:18,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:18,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:18,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:19,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159639008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159639008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159639038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159639048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:26:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:19,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:19,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/566cd56a45b64bd3a69a5c99cfa2cd3f 2024-11-21T03:26:19,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/3d0d7f2b42a8465d8c75b2ab167d6989 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3d0d7f2b42a8465d8c75b2ab167d6989 2024-11-21T03:26:19,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3d0d7f2b42a8465d8c75b2ab167d6989, entries=200, sequenceid=423, filesize=14.4 K 2024-11-21T03:26:19,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/4551035bb2c14279bf77cd60c607fc74 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4551035bb2c14279bf77cd60c607fc74 2024-11-21T03:26:19,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4551035bb2c14279bf77cd60c607fc74, entries=150, sequenceid=423, filesize=12.0 K 2024-11-21T03:26:19,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/566cd56a45b64bd3a69a5c99cfa2cd3f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/566cd56a45b64bd3a69a5c99cfa2cd3f 2024-11-21T03:26:19,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/566cd56a45b64bd3a69a5c99cfa2cd3f, entries=150, sequenceid=423, filesize=12.0 K 2024-11-21T03:26:19,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 215fc242d64dec572e979318d587db2b in 1043ms, sequenceid=423, compaction requested=true 2024-11-21T03:26:19,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:19,264 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:19,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:19,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:19,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:19,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:19,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:19,265 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:19,265 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:19,265 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:19,265 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f39c5a203924d9cacef6c11f48b7e2c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/38c15e05651d45928388369ebfbcbbaa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3d0d7f2b42a8465d8c75b2ab167d6989] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=39.4 K 2024-11-21T03:26:19,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:26:19,265 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f39c5a203924d9cacef6c11f48b7e2c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732159576445 2024-11-21T03:26:19,266 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c15e05651d45928388369ebfbcbbaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732159576838 2024-11-21T03:26:19,266 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d0d7f2b42a8465d8c75b2ab167d6989, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732159578085 2024-11-21T03:26:19,267 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:19,280 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:19,280 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:19,280 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:19,280 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/dc866f60c1d842048a7bac66bf64f9c8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/1ce2202fe3ef44e5b38a2ac65d2fc85b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4551035bb2c14279bf77cd60c607fc74] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=37.0 K 2024-11-21T03:26:19,282 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:19,282 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/9ffe67b55d20415fbcbcf038dbbde1d8 is 50, key is test_row_0/A:col10/1732159578085/Put/seqid=0 2024-11-21T03:26:19,282 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting dc866f60c1d842048a7bac66bf64f9c8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732159576445 2024-11-21T03:26:19,284 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ce2202fe3ef44e5b38a2ac65d2fc85b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732159576838 2024-11-21T03:26:19,284 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4551035bb2c14279bf77cd60c607fc74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732159578085 2024-11-21T03:26:19,298 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#493 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:19,298 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/91e161917e42470d86f2c8811e5aa471 is 50, key is test_row_0/B:col10/1732159578085/Put/seqid=0 2024-11-21T03:26:19,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:26:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:19,325 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T03:26:19,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742396_1572 (size=13357) 2024-11-21T03:26:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:26:19,338 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/9ffe67b55d20415fbcbcf038dbbde1d8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9ffe67b55d20415fbcbcf038dbbde1d8 2024-11-21T03:26:19,344 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into 9ffe67b55d20415fbcbcf038dbbde1d8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:19,344 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:19,344 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=13, startTime=1732159579264; duration=0sec 2024-11-21T03:26:19,344 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:19,344 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:19,344 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:19,345 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:19,345 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:19,345 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:19,345 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a661f59a81f54ac4ae4099ddd895b577, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a1f70a947a4a404e980e62f2e202de5b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/566cd56a45b64bd3a69a5c99cfa2cd3f] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=37.0 K 2024-11-21T03:26:19,345 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a661f59a81f54ac4ae4099ddd895b577, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732159576445 2024-11-21T03:26:19,346 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f70a947a4a404e980e62f2e202de5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732159576838 2024-11-21T03:26:19,346 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 566cd56a45b64bd3a69a5c99cfa2cd3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732159578085 2024-11-21T03:26:19,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742397_1573 (size=13357) 2024-11-21T03:26:19,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/852078fa681d4e439433ba2500094329 is 50, key is test_row_0/A:col10/1732159578352/Put/seqid=0 2024-11-21T03:26:19,370 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:19,371 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/494ccebf255c46e791718576a36ec54a is 50, key is test_row_0/C:col10/1732159578085/Put/seqid=0 2024-11-21T03:26:19,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742398_1574 (size=12301) 2024-11-21T03:26:19,408 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/852078fa681d4e439433ba2500094329 2024-11-21T03:26:19,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742399_1575 (size=13357) 2024-11-21T03:26:19,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/8aed253bee904948907965cda5fae92e is 50, key is test_row_0/B:col10/1732159578352/Put/seqid=0 2024-11-21T03:26:19,428 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/494ccebf255c46e791718576a36ec54a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/494ccebf255c46e791718576a36ec54a 2024-11-21T03:26:19,448 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into 494ccebf255c46e791718576a36ec54a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:19,448 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:19,448 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=13, startTime=1732159579264; duration=0sec 2024-11-21T03:26:19,448 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:19,448 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:19,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742400_1576 (size=12301) 2024-11-21T03:26:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:19,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159639599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159639610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159639614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159639615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159639720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159639720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159639737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159639739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,810 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/91e161917e42470d86f2c8811e5aa471 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/91e161917e42470d86f2c8811e5aa471 2024-11-21T03:26:19,859 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/8aed253bee904948907965cda5fae92e 2024-11-21T03:26:19,864 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into 91e161917e42470d86f2c8811e5aa471(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:19,864 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:19,864 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=13, startTime=1732159579264; duration=0sec 2024-11-21T03:26:19,864 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:19,864 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:19,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/3b5fc7ec4bc844c1b9a7e1be6aa9a316 is 50, key is test_row_0/C:col10/1732159578352/Put/seqid=0 2024-11-21T03:26:19,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159639938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742401_1577 (size=12301) 2024-11-21T03:26:19,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159639948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,964 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/3b5fc7ec4bc844c1b9a7e1be6aa9a316 2024-11-21T03:26:19,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159639960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159639961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/852078fa681d4e439433ba2500094329 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/852078fa681d4e439433ba2500094329 2024-11-21T03:26:20,069 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/852078fa681d4e439433ba2500094329, entries=150, sequenceid=444, filesize=12.0 K 2024-11-21T03:26:20,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/8aed253bee904948907965cda5fae92e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8aed253bee904948907965cda5fae92e 2024-11-21T03:26:20,129 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8aed253bee904948907965cda5fae92e, entries=150, sequenceid=444, filesize=12.0 K 2024-11-21T03:26:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/3b5fc7ec4bc844c1b9a7e1be6aa9a316 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3b5fc7ec4bc844c1b9a7e1be6aa9a316 2024-11-21T03:26:20,141 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3b5fc7ec4bc844c1b9a7e1be6aa9a316, entries=150, sequenceid=444, filesize=12.0 K 2024-11-21T03:26:20,142 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 215fc242d64dec572e979318d587db2b in 817ms, sequenceid=444, compaction requested=false 2024-11-21T03:26:20,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:20,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:20,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-21T03:26:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-21T03:26:20,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-21T03:26:20,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-11-21T03:26:20,151 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9490 sec 2024-11-21T03:26:20,269 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:20,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T03:26:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:20,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/27e4f7afa1274041b3b54b1d32e73a63 is 50, key is test_row_0/A:col10/1732159579612/Put/seqid=0 2024-11-21T03:26:20,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742402_1578 (size=14741) 2024-11-21T03:26:20,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/27e4f7afa1274041b3b54b1d32e73a63 2024-11-21T03:26:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:26:20,349 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:20,349 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-21T03:26:20,351 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T03:26:20,352 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:20,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:20,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/934f525a394d4263a639375e040bdcbb is 50, key is test_row_0/B:col10/1732159579612/Put/seqid=0 2024-11-21T03:26:20,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159640346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159640358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159640359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159640363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742403_1579 (size=12301) 2024-11-21T03:26:20,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/934f525a394d4263a639375e040bdcbb 2024-11-21T03:26:20,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/6779958035d1475887f5068f72f2ed80 is 50, key is test_row_0/C:col10/1732159579612/Put/seqid=0 2024-11-21T03:26:20,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742404_1580 (size=12301) 2024-11-21T03:26:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T03:26:20,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159640469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159640478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T03:26:20,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:20,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:20,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:20,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:20,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:20,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159640489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159640489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T03:26:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:20,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:20,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:20,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:20,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T03:26:20,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49356 deadline: 1732159640688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49414 deadline: 1732159640698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49374 deadline: 1732159640717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49388 deadline: 1732159640720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49400 deadline: 1732159640758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:20,771 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in 215fc242d64dec572e979318d587db2b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:26:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 , the old value is region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., hostname=b69e2a1ee061,45649,1732159441548, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:26:20,818 INFO [Thread-2147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2147. 2024-11-21T03:26:20,818 DEBUG [Thread-2147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:20,818 DEBUG [Thread-2147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:20,818 DEBUG [Thread-2147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:20,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:20,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T03:26:20,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:20,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:20,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:20,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:20,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:20,823 INFO [Thread-2149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2149. 2024-11-21T03:26:20,823 DEBUG [Thread-2149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:20,823 DEBUG [Thread-2149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:20,823 DEBUG [Thread-2149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:20,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:20,834 INFO [Thread-2145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2145. 2024-11-21T03:26:20,834 DEBUG [Thread-2145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:20,834 DEBUG [Thread-2145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:20,834 DEBUG [Thread-2145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:20,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:20,843 INFO [Thread-2151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2151. 2024-11-21T03:26:20,843 DEBUG [Thread-2151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:20,843 DEBUG [Thread-2151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:20,844 DEBUG [Thread-2151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:20,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:20,847 INFO [Thread-2143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2143. 2024-11-21T03:26:20,847 DEBUG [Thread-2143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:20,847 DEBUG [Thread-2143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:20,847 DEBUG [Thread-2143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:20,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:20,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/6779958035d1475887f5068f72f2ed80 2024-11-21T03:26:20,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/27e4f7afa1274041b3b54b1d32e73a63 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/27e4f7afa1274041b3b54b1d32e73a63 2024-11-21T03:26:20,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/27e4f7afa1274041b3b54b1d32e73a63, entries=200, sequenceid=464, filesize=14.4 K 2024-11-21T03:26:20,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/934f525a394d4263a639375e040bdcbb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/934f525a394d4263a639375e040bdcbb 2024-11-21T03:26:20,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/934f525a394d4263a639375e040bdcbb, entries=150, sequenceid=464, filesize=12.0 K 2024-11-21T03:26:20,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/6779958035d1475887f5068f72f2ed80 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/6779958035d1475887f5068f72f2ed80 2024-11-21T03:26:20,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/6779958035d1475887f5068f72f2ed80, entries=150, sequenceid=464, filesize=12.0 K 2024-11-21T03:26:20,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 215fc242d64dec572e979318d587db2b in 633ms, sequenceid=464, compaction requested=true 2024-11-21T03:26:20,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:20,902 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:20,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:20,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:20,903 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:20,903 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:20,903 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/A is initiating minor compaction (all files) 2024-11-21T03:26:20,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:20,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:20,903 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/A in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:20,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 215fc242d64dec572e979318d587db2b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:20,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:20,903 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9ffe67b55d20415fbcbcf038dbbde1d8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/852078fa681d4e439433ba2500094329, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/27e4f7afa1274041b3b54b1d32e73a63] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=39.5 K 2024-11-21T03:26:20,904 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ffe67b55d20415fbcbcf038dbbde1d8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732159578085 2024-11-21T03:26:20,908 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 852078fa681d4e439433ba2500094329, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732159578291 2024-11-21T03:26:20,909 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e4f7afa1274041b3b54b1d32e73a63, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732159579595 2024-11-21T03:26:20,911 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:20,911 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/B is initiating minor compaction (all files) 2024-11-21T03:26:20,911 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/B in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:20,911 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/91e161917e42470d86f2c8811e5aa471, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8aed253bee904948907965cda5fae92e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/934f525a394d4263a639375e040bdcbb] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=37.1 K 2024-11-21T03:26:20,912 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 91e161917e42470d86f2c8811e5aa471, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732159578085 2024-11-21T03:26:20,913 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aed253bee904948907965cda5fae92e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732159578291 2024-11-21T03:26:20,917 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 934f525a394d4263a639375e040bdcbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732159579612 2024-11-21T03:26:20,932 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#A#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:20,933 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/695237b4f2284385856973fb993e579c is 50, key is test_row_0/A:col10/1732159579612/Put/seqid=0 2024-11-21T03:26:20,950 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:20,951 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/328d3775356640fda94091d8ace71e9a is 50, key is test_row_0/B:col10/1732159579612/Put/seqid=0 2024-11-21T03:26:20,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742405_1581 (size=13459) 2024-11-21T03:26:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T03:26:20,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T03:26:20,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:20,982 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T03:26:20,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742406_1582 (size=13459) 2024-11-21T03:26:20,993 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/328d3775356640fda94091d8ace71e9a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/328d3775356640fda94091d8ace71e9a 2024-11-21T03:26:21,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/499d2726530d4564881e370b1dd6f70d is 50, key is test_row_0/A:col10/1732159580360/Put/seqid=0 2024-11-21T03:26:21,007 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/B of 215fc242d64dec572e979318d587db2b into 328d3775356640fda94091d8ace71e9a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:21,007 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:21,007 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/B, priority=13, startTime=1732159580903; duration=0sec 2024-11-21T03:26:21,007 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:21,007 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:B 2024-11-21T03:26:21,007 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:21,008 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:21,008 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 215fc242d64dec572e979318d587db2b/C is initiating minor compaction (all files) 2024-11-21T03:26:21,009 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 215fc242d64dec572e979318d587db2b/C in TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:21,009 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/494ccebf255c46e791718576a36ec54a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3b5fc7ec4bc844c1b9a7e1be6aa9a316, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/6779958035d1475887f5068f72f2ed80] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp, totalSize=37.1 K 2024-11-21T03:26:21,009 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 494ccebf255c46e791718576a36ec54a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732159578085 2024-11-21T03:26:21,010 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b5fc7ec4bc844c1b9a7e1be6aa9a316, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732159578291 2024-11-21T03:26:21,010 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6779958035d1475887f5068f72f2ed80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732159579612 2024-11-21T03:26:21,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742407_1583 (size=12301) 2024-11-21T03:26:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:21,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. as already flushing 2024-11-21T03:26:21,020 INFO [Thread-2140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2140. 2024-11-21T03:26:21,020 DEBUG [Thread-2140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:21,020 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:21,020 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:21,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:21,021 INFO [Thread-2138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2138. 2024-11-21T03:26:21,021 DEBUG [Thread-2138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:21,021 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:21,021 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:21,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:21,023 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 215fc242d64dec572e979318d587db2b#C#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:21,024 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/928eca51b94c46c3be1f001c71f4a65f is 50, key is test_row_0/C:col10/1732159579612/Put/seqid=0 2024-11-21T03:26:21,029 INFO [Thread-2134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2134. 2024-11-21T03:26:21,029 DEBUG [Thread-2134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:21,029 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:21,029 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:21,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:21,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742408_1584 (size=13459) 2024-11-21T03:26:21,037 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/928eca51b94c46c3be1f001c71f4a65f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/928eca51b94c46c3be1f001c71f4a65f 2024-11-21T03:26:21,043 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/C of 215fc242d64dec572e979318d587db2b into 928eca51b94c46c3be1f001c71f4a65f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:21,043 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:21,043 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/C, priority=13, startTime=1732159580903; duration=0sec 2024-11-21T03:26:21,043 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:21,043 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:C 2024-11-21T03:26:21,050 INFO [Thread-2132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2132. 2024-11-21T03:26:21,050 DEBUG [Thread-2132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:21,050 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:21,050 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:21,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:21,373 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/695237b4f2284385856973fb993e579c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/695237b4f2284385856973fb993e579c 2024-11-21T03:26:21,377 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 215fc242d64dec572e979318d587db2b/A of 215fc242d64dec572e979318d587db2b into 695237b4f2284385856973fb993e579c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:21,377 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:21,377 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., storeName=215fc242d64dec572e979318d587db2b/A, priority=13, startTime=1732159580902; duration=0sec 2024-11-21T03:26:21,377 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:21,377 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 215fc242d64dec572e979318d587db2b:A 2024-11-21T03:26:21,412 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/499d2726530d4564881e370b1dd6f70d 2024-11-21T03:26:21,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/5bc0c3926ffa471e9a7a4c46e18c2f4d is 50, key is test_row_0/B:col10/1732159580360/Put/seqid=0 2024-11-21T03:26:21,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742409_1585 (size=12301) 2024-11-21T03:26:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T03:26:21,836 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/5bc0c3926ffa471e9a7a4c46e18c2f4d 2024-11-21T03:26:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/21d7a2c3fd8c4f19857686180b3a94c3 is 50, key is test_row_0/C:col10/1732159580360/Put/seqid=0 2024-11-21T03:26:21,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742410_1586 (size=12301) 2024-11-21T03:26:21,884 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/21d7a2c3fd8c4f19857686180b3a94c3 2024-11-21T03:26:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/499d2726530d4564881e370b1dd6f70d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/499d2726530d4564881e370b1dd6f70d 2024-11-21T03:26:21,899 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/499d2726530d4564881e370b1dd6f70d, entries=150, sequenceid=483, filesize=12.0 K 2024-11-21T03:26:21,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/5bc0c3926ffa471e9a7a4c46e18c2f4d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/5bc0c3926ffa471e9a7a4c46e18c2f4d 2024-11-21T03:26:21,909 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/5bc0c3926ffa471e9a7a4c46e18c2f4d, entries=150, sequenceid=483, filesize=12.0 K 2024-11-21T03:26:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/21d7a2c3fd8c4f19857686180b3a94c3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/21d7a2c3fd8c4f19857686180b3a94c3 2024-11-21T03:26:21,918 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/21d7a2c3fd8c4f19857686180b3a94c3, entries=150, sequenceid=483, filesize=12.0 K 2024-11-21T03:26:21,919 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=26.84 KB/27480 for 215fc242d64dec572e979318d587db2b in 937ms, sequenceid=483, compaction requested=false 2024-11-21T03:26:21,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 215fc242d64dec572e979318d587db2b: 2024-11-21T03:26:21,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:21,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-21T03:26:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-21T03:26:21,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-21T03:26:21,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-21T03:26:21,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5750 sec 2024-11-21T03:26:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T03:26:22,508 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:24,790 INFO [Thread-2136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2136. 2024-11-21T03:26:24,790 DEBUG [Thread-2136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:24,790 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:24,790 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:24,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 989 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2967 rows 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 980 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2940 rows 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 989 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2964 rows 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 990 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2970 rows 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 980 2024-11-21T03:26:24,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2937 rows 2024-11-21T03:26:24,791 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:26:24,792 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:26:24,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:24,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:24,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:24,793 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T03:26:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T03:26:24,797 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159584797"}]},"ts":"1732159584797"} 2024-11-21T03:26:24,799 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T03:26:24,799 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T03:26:24,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:26:24,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=215fc242d64dec572e979318d587db2b, UNASSIGN}] 2024-11-21T03:26:24,803 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=215fc242d64dec572e979318d587db2b, UNASSIGN 2024-11-21T03:26:24,804 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=215fc242d64dec572e979318d587db2b, regionState=CLOSING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:24,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=215fc242d64dec572e979318d587db2b, UNASSIGN because future has completed 2024-11-21T03:26:24,807 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:26:24,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:26:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T03:26:24,959 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:24,959 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:26:24,960 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 215fc242d64dec572e979318d587db2b, disabling compactions & flushes 2024-11-21T03:26:24,960 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:24,960 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:24,960 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. after waiting 0 ms 2024-11-21T03:26:24,960 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:24,960 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 215fc242d64dec572e979318d587db2b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T03:26:24,963 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/afcc42c1312940218c927eff70d4a30d is 50, key is test_row_1/A:col10/1732159581018/Put/seqid=0 2024-11-21T03:26:24,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742411_1587 (size=9857) 2024-11-21T03:26:24,986 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/afcc42c1312940218c927eff70d4a30d 2024-11-21T03:26:24,997 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/e86ab7dbe46441cb8b9dbd93af0105fc is 50, key is test_row_1/B:col10/1732159581018/Put/seqid=0 2024-11-21T03:26:25,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742412_1588 (size=9857) 2024-11-21T03:26:25,022 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/e86ab7dbe46441cb8b9dbd93af0105fc 2024-11-21T03:26:25,028 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/7842b8173a46441688a4f722a25a9c57 is 50, key is test_row_1/C:col10/1732159581018/Put/seqid=0 2024-11-21T03:26:25,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742413_1589 (size=9857) 2024-11-21T03:26:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T03:26:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T03:26:25,442 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/7842b8173a46441688a4f722a25a9c57 2024-11-21T03:26:25,447 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/A/afcc42c1312940218c927eff70d4a30d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/afcc42c1312940218c927eff70d4a30d 2024-11-21T03:26:25,454 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/afcc42c1312940218c927eff70d4a30d, entries=100, sequenceid=494, filesize=9.6 K 2024-11-21T03:26:25,455 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/B/e86ab7dbe46441cb8b9dbd93af0105fc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e86ab7dbe46441cb8b9dbd93af0105fc 2024-11-21T03:26:25,459 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e86ab7dbe46441cb8b9dbd93af0105fc, entries=100, sequenceid=494, filesize=9.6 K 2024-11-21T03:26:25,460 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/.tmp/C/7842b8173a46441688a4f722a25a9c57 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/7842b8173a46441688a4f722a25a9c57 2024-11-21T03:26:25,464 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/7842b8173a46441688a4f722a25a9c57, entries=100, sequenceid=494, filesize=9.6 K 2024-11-21T03:26:25,465 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 215fc242d64dec572e979318d587db2b in 505ms, sequenceid=494, compaction requested=true 2024-11-21T03:26:25,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/642116801803413c9fbdd483f86d4d36, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f0d93da9f6745418ecd111fefcfbddd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/a81599aa4f7b4880875222247de618bd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/19b18c67f83841088cc76f78cbf0065a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5dfd9d0b421e4fd5bf0ab1e951b40b7d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/aedb5b24625b4339982c7eca2b86b2fc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/83a980bc371647f4b40708f992ebff5b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/0588850dda3a48d9a8051808718f52e1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/afd3c594d4d74f43aa1583958be83e68, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/d5a6709f6cf14dbf8e881db67205ca01, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/f515c7876e4742c481b5389544b972ba, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/94093bfa59d94790a8b7b1af0b511c91, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5a1dcb3dd82041f5b77422dcc948510e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9cc348d8138b49a79d48b7cf6bdd263a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6732a41314fa461bbef6e7377a97273c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/856d3b361282415eaae07d79d3972604, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/91312763d71a4e3b8eb69fbd2b459a1c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/ce843858b1204f5eab165241ae678964, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3033db9f7afc44a4bc9ee2405e6a54a4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/262f0829c422440b8e0cf943202ed0e4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6a0577f37b7d4fa39cc53ee8ec111c51, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4eb7b7863a354896b44616d12aacee81, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/e370bcc5ec414abdaf2f0794a26c0ea0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9e42a957026d4ec58624934b316e03aa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4e9af5059ebe46578eed7e87a4da8478, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9a9f49a1ffa04c3897d04e22a49ddee5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/b31848967af94722a926d6c4a9d477d5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/766ef235739c4d5db487a6daef68334e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f39c5a203924d9cacef6c11f48b7e2c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/38c15e05651d45928388369ebfbcbbaa, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3d0d7f2b42a8465d8c75b2ab167d6989, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9ffe67b55d20415fbcbcf038dbbde1d8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/852078fa681d4e439433ba2500094329, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/27e4f7afa1274041b3b54b1d32e73a63] to archive 2024-11-21T03:26:25,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:26:25,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/642116801803413c9fbdd483f86d4d36 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/642116801803413c9fbdd483f86d4d36 2024-11-21T03:26:25,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f0d93da9f6745418ecd111fefcfbddd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f0d93da9f6745418ecd111fefcfbddd 2024-11-21T03:26:25,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/a81599aa4f7b4880875222247de618bd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/a81599aa4f7b4880875222247de618bd 2024-11-21T03:26:25,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/19b18c67f83841088cc76f78cbf0065a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/19b18c67f83841088cc76f78cbf0065a 2024-11-21T03:26:25,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5dfd9d0b421e4fd5bf0ab1e951b40b7d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5dfd9d0b421e4fd5bf0ab1e951b40b7d 2024-11-21T03:26:25,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/aedb5b24625b4339982c7eca2b86b2fc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/aedb5b24625b4339982c7eca2b86b2fc 2024-11-21T03:26:25,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/83a980bc371647f4b40708f992ebff5b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/83a980bc371647f4b40708f992ebff5b 2024-11-21T03:26:25,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/0588850dda3a48d9a8051808718f52e1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/0588850dda3a48d9a8051808718f52e1 2024-11-21T03:26:25,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/afd3c594d4d74f43aa1583958be83e68 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/afd3c594d4d74f43aa1583958be83e68 2024-11-21T03:26:25,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/d5a6709f6cf14dbf8e881db67205ca01 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/d5a6709f6cf14dbf8e881db67205ca01 2024-11-21T03:26:25,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/f515c7876e4742c481b5389544b972ba to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/f515c7876e4742c481b5389544b972ba 2024-11-21T03:26:25,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/94093bfa59d94790a8b7b1af0b511c91 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/94093bfa59d94790a8b7b1af0b511c91 2024-11-21T03:26:25,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5a1dcb3dd82041f5b77422dcc948510e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/5a1dcb3dd82041f5b77422dcc948510e 2024-11-21T03:26:25,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9cc348d8138b49a79d48b7cf6bdd263a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9cc348d8138b49a79d48b7cf6bdd263a 2024-11-21T03:26:25,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6732a41314fa461bbef6e7377a97273c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6732a41314fa461bbef6e7377a97273c 2024-11-21T03:26:25,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/856d3b361282415eaae07d79d3972604 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/856d3b361282415eaae07d79d3972604 2024-11-21T03:26:25,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/91312763d71a4e3b8eb69fbd2b459a1c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/91312763d71a4e3b8eb69fbd2b459a1c 2024-11-21T03:26:25,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/ce843858b1204f5eab165241ae678964 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/ce843858b1204f5eab165241ae678964 2024-11-21T03:26:25,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3033db9f7afc44a4bc9ee2405e6a54a4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3033db9f7afc44a4bc9ee2405e6a54a4 2024-11-21T03:26:25,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/262f0829c422440b8e0cf943202ed0e4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/262f0829c422440b8e0cf943202ed0e4 2024-11-21T03:26:25,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6a0577f37b7d4fa39cc53ee8ec111c51 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/6a0577f37b7d4fa39cc53ee8ec111c51 2024-11-21T03:26:25,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4eb7b7863a354896b44616d12aacee81 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4eb7b7863a354896b44616d12aacee81 2024-11-21T03:26:25,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/e370bcc5ec414abdaf2f0794a26c0ea0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/e370bcc5ec414abdaf2f0794a26c0ea0 2024-11-21T03:26:25,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9e42a957026d4ec58624934b316e03aa to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9e42a957026d4ec58624934b316e03aa 2024-11-21T03:26:25,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4e9af5059ebe46578eed7e87a4da8478 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/4e9af5059ebe46578eed7e87a4da8478 2024-11-21T03:26:25,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9a9f49a1ffa04c3897d04e22a49ddee5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9a9f49a1ffa04c3897d04e22a49ddee5 2024-11-21T03:26:25,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/b31848967af94722a926d6c4a9d477d5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/b31848967af94722a926d6c4a9d477d5 2024-11-21T03:26:25,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/766ef235739c4d5db487a6daef68334e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/766ef235739c4d5db487a6daef68334e 2024-11-21T03:26:25,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f39c5a203924d9cacef6c11f48b7e2c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/1f39c5a203924d9cacef6c11f48b7e2c 2024-11-21T03:26:25,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/38c15e05651d45928388369ebfbcbbaa to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/38c15e05651d45928388369ebfbcbbaa 2024-11-21T03:26:25,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3d0d7f2b42a8465d8c75b2ab167d6989 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/3d0d7f2b42a8465d8c75b2ab167d6989 2024-11-21T03:26:25,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9ffe67b55d20415fbcbcf038dbbde1d8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/9ffe67b55d20415fbcbcf038dbbde1d8 2024-11-21T03:26:25,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/852078fa681d4e439433ba2500094329 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/852078fa681d4e439433ba2500094329 2024-11-21T03:26:25,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/27e4f7afa1274041b3b54b1d32e73a63 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/27e4f7afa1274041b3b54b1d32e73a63 2024-11-21T03:26:25,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/f22cdff474c1476da7c4985d2e850f4a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c747bf6c55f0476c9ce09e1de1bffd4d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/5d1d3eaaa62e4097b3dbe1c52f01d8f4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/93f6fc8e43844297bdfcf85b92396cb2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c6d157f2a58f4973bb48b07ce3f2d767, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c9079b1204ea4fe39a3dfd41ad430ddf, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d20e9bbb707545a595218b44ee3759a2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/90810f53c6c84f1b97d1a31e0297f369, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c61d8555db204d68bf64d3003adc7f36, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0be37e921f404e409aaf3243cfe982be, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e19bb53f34504778ab4ed648be06e5ac, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4f1d4e0e5e744d949fffb5031f7e6fdf, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/b625de4b4331432ca1f4a015265c9f4d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4de7c3f207a1455192f13d332d18f65c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0455dd8ae13f4987a702bdc4bd26b95b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/3bcc98d13f5347738dd535e35b2661b1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/84559797c02440ad9373381fd97c638b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d6524873b51341b0bd6e6d72f58054a5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e314454badbe468e8113cf11da3f2748, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/fd9b2a2fc37b4918a63a4d77861bf8e3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8cdbb89190b442c3b5c0a3b289efb913, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/b5d1de263efb4eb8ae8c528ddc1d215e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/6057852be6dd4bcab9875d1e8e63da18, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c5ef37c7771b479aae0c312a4020e421, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0d819a74932f4928b9da8ef2c10e8641, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/51029712c89642a987f9d6c164da8665, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4067427061674aac9cd9e8d204bbc233, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/dc866f60c1d842048a7bac66bf64f9c8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/83954f56af1440a996a01b3415f3fffe, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/1ce2202fe3ef44e5b38a2ac65d2fc85b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/91e161917e42470d86f2c8811e5aa471, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4551035bb2c14279bf77cd60c607fc74, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8aed253bee904948907965cda5fae92e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/934f525a394d4263a639375e040bdcbb] to archive 2024-11-21T03:26:25,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:26:25,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/f22cdff474c1476da7c4985d2e850f4a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/f22cdff474c1476da7c4985d2e850f4a 2024-11-21T03:26:25,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c747bf6c55f0476c9ce09e1de1bffd4d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c747bf6c55f0476c9ce09e1de1bffd4d 2024-11-21T03:26:25,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/5d1d3eaaa62e4097b3dbe1c52f01d8f4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/5d1d3eaaa62e4097b3dbe1c52f01d8f4 2024-11-21T03:26:25,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/93f6fc8e43844297bdfcf85b92396cb2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/93f6fc8e43844297bdfcf85b92396cb2 2024-11-21T03:26:25,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c6d157f2a58f4973bb48b07ce3f2d767 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c6d157f2a58f4973bb48b07ce3f2d767 2024-11-21T03:26:25,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c9079b1204ea4fe39a3dfd41ad430ddf to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c9079b1204ea4fe39a3dfd41ad430ddf 2024-11-21T03:26:25,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d20e9bbb707545a595218b44ee3759a2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d20e9bbb707545a595218b44ee3759a2 2024-11-21T03:26:25,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/90810f53c6c84f1b97d1a31e0297f369 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/90810f53c6c84f1b97d1a31e0297f369 2024-11-21T03:26:25,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c61d8555db204d68bf64d3003adc7f36 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c61d8555db204d68bf64d3003adc7f36 2024-11-21T03:26:25,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0be37e921f404e409aaf3243cfe982be to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0be37e921f404e409aaf3243cfe982be 2024-11-21T03:26:25,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e19bb53f34504778ab4ed648be06e5ac to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e19bb53f34504778ab4ed648be06e5ac 2024-11-21T03:26:25,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4f1d4e0e5e744d949fffb5031f7e6fdf to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4f1d4e0e5e744d949fffb5031f7e6fdf 2024-11-21T03:26:25,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/b625de4b4331432ca1f4a015265c9f4d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/b625de4b4331432ca1f4a015265c9f4d 2024-11-21T03:26:25,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4de7c3f207a1455192f13d332d18f65c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4de7c3f207a1455192f13d332d18f65c 2024-11-21T03:26:25,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0455dd8ae13f4987a702bdc4bd26b95b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0455dd8ae13f4987a702bdc4bd26b95b 2024-11-21T03:26:25,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/3bcc98d13f5347738dd535e35b2661b1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/3bcc98d13f5347738dd535e35b2661b1 2024-11-21T03:26:25,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/84559797c02440ad9373381fd97c638b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/84559797c02440ad9373381fd97c638b 2024-11-21T03:26:25,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d6524873b51341b0bd6e6d72f58054a5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/d6524873b51341b0bd6e6d72f58054a5 2024-11-21T03:26:25,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e314454badbe468e8113cf11da3f2748 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e314454badbe468e8113cf11da3f2748 2024-11-21T03:26:25,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/fd9b2a2fc37b4918a63a4d77861bf8e3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/fd9b2a2fc37b4918a63a4d77861bf8e3 2024-11-21T03:26:25,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8cdbb89190b442c3b5c0a3b289efb913 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8cdbb89190b442c3b5c0a3b289efb913 2024-11-21T03:26:25,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/b5d1de263efb4eb8ae8c528ddc1d215e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/b5d1de263efb4eb8ae8c528ddc1d215e 2024-11-21T03:26:25,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/6057852be6dd4bcab9875d1e8e63da18 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/6057852be6dd4bcab9875d1e8e63da18 2024-11-21T03:26:25,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c5ef37c7771b479aae0c312a4020e421 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/c5ef37c7771b479aae0c312a4020e421 2024-11-21T03:26:25,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0d819a74932f4928b9da8ef2c10e8641 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/0d819a74932f4928b9da8ef2c10e8641 2024-11-21T03:26:25,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/51029712c89642a987f9d6c164da8665 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/51029712c89642a987f9d6c164da8665 2024-11-21T03:26:25,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4067427061674aac9cd9e8d204bbc233 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4067427061674aac9cd9e8d204bbc233 2024-11-21T03:26:25,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/dc866f60c1d842048a7bac66bf64f9c8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/dc866f60c1d842048a7bac66bf64f9c8 2024-11-21T03:26:25,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/83954f56af1440a996a01b3415f3fffe to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/83954f56af1440a996a01b3415f3fffe 2024-11-21T03:26:25,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/1ce2202fe3ef44e5b38a2ac65d2fc85b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/1ce2202fe3ef44e5b38a2ac65d2fc85b 2024-11-21T03:26:25,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/91e161917e42470d86f2c8811e5aa471 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/91e161917e42470d86f2c8811e5aa471 2024-11-21T03:26:25,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4551035bb2c14279bf77cd60c607fc74 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/4551035bb2c14279bf77cd60c607fc74 2024-11-21T03:26:25,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8aed253bee904948907965cda5fae92e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/8aed253bee904948907965cda5fae92e 2024-11-21T03:26:25,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/934f525a394d4263a639375e040bdcbb to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/934f525a394d4263a639375e040bdcbb 2024-11-21T03:26:25,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fcb33bff28a54e9eaf906f22229d1267, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/cd028d5b8fff44a8b0a7a2eb05006001, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/93fe930013b94cc19417d7fb57f305dd, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/239d480c8bbd442d8205acefe57b702d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/41cd49f7a54f40ef9252464230511cc4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3e738b6c532c4bcbb78cc5497ede8147, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/f0760736e07d46518a0ee0425ea1dc7d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bddf570f18e14db3b665c45fc933bb09, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/429a3c64d3f64a3b8d27455c97552007, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/056c3b2c07b9409b919c3a498973a3c3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/c535a44594eb468c92595ae714b09304, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/2a266fdc7bc8481891b804873dd4d5b1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4fe80342ca9345be876d9f7f3e72cf1f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/b13cdb792a4640498f065b736e361099, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/89f8af56b1d34da3a24daf5a35226197, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/f9cefee148a94247bc888c77aec46981, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bebbe0cb4cfb4471a0b2efac93b73d02, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/8ee19f2e35a04ea0b97f937af8557e4b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/55608e4e18484acaa778ea3e35ac6784, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4a426b6d72cc4ab1880a1d4cd3fdf7db, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/e9d2b70bc0434a7aaf6fee0ad1b5af03, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/9a9cc1728b7d406398ffefd68c76e491, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4dfa975b225b4c14b2a1d46e207cad2a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/99ef0377c98641a981b4a770fea35785, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/628e2612b04548859ca0e9028eb69788, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fe26d1d83298489289340a8df6ef2e26, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/96044d9b697f4ccba7c4be3c4d9762b1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a661f59a81f54ac4ae4099ddd895b577, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/0186fc2388af4c0ca83672ac912994c0, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a1f70a947a4a404e980e62f2e202de5b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/494ccebf255c46e791718576a36ec54a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/566cd56a45b64bd3a69a5c99cfa2cd3f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3b5fc7ec4bc844c1b9a7e1be6aa9a316, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/6779958035d1475887f5068f72f2ed80] to archive 2024-11-21T03:26:25,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:26:25,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fcb33bff28a54e9eaf906f22229d1267 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fcb33bff28a54e9eaf906f22229d1267 2024-11-21T03:26:25,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/cd028d5b8fff44a8b0a7a2eb05006001 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/cd028d5b8fff44a8b0a7a2eb05006001 2024-11-21T03:26:25,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/93fe930013b94cc19417d7fb57f305dd to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/93fe930013b94cc19417d7fb57f305dd 2024-11-21T03:26:25,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/239d480c8bbd442d8205acefe57b702d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/239d480c8bbd442d8205acefe57b702d 2024-11-21T03:26:25,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/41cd49f7a54f40ef9252464230511cc4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/41cd49f7a54f40ef9252464230511cc4 2024-11-21T03:26:25,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3e738b6c532c4bcbb78cc5497ede8147 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3e738b6c532c4bcbb78cc5497ede8147 2024-11-21T03:26:25,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/f0760736e07d46518a0ee0425ea1dc7d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/f0760736e07d46518a0ee0425ea1dc7d 2024-11-21T03:26:25,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bddf570f18e14db3b665c45fc933bb09 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bddf570f18e14db3b665c45fc933bb09 2024-11-21T03:26:25,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/429a3c64d3f64a3b8d27455c97552007 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/429a3c64d3f64a3b8d27455c97552007 2024-11-21T03:26:25,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/056c3b2c07b9409b919c3a498973a3c3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/056c3b2c07b9409b919c3a498973a3c3 2024-11-21T03:26:25,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/c535a44594eb468c92595ae714b09304 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/c535a44594eb468c92595ae714b09304 2024-11-21T03:26:25,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/2a266fdc7bc8481891b804873dd4d5b1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/2a266fdc7bc8481891b804873dd4d5b1 2024-11-21T03:26:25,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4fe80342ca9345be876d9f7f3e72cf1f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4fe80342ca9345be876d9f7f3e72cf1f 2024-11-21T03:26:25,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/b13cdb792a4640498f065b736e361099 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/b13cdb792a4640498f065b736e361099 2024-11-21T03:26:25,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/89f8af56b1d34da3a24daf5a35226197 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/89f8af56b1d34da3a24daf5a35226197 2024-11-21T03:26:25,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/f9cefee148a94247bc888c77aec46981 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/f9cefee148a94247bc888c77aec46981 2024-11-21T03:26:25,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bebbe0cb4cfb4471a0b2efac93b73d02 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/bebbe0cb4cfb4471a0b2efac93b73d02 2024-11-21T03:26:25,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/8ee19f2e35a04ea0b97f937af8557e4b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/8ee19f2e35a04ea0b97f937af8557e4b 2024-11-21T03:26:25,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/55608e4e18484acaa778ea3e35ac6784 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/55608e4e18484acaa778ea3e35ac6784 2024-11-21T03:26:25,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4a426b6d72cc4ab1880a1d4cd3fdf7db to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4a426b6d72cc4ab1880a1d4cd3fdf7db 2024-11-21T03:26:25,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/e9d2b70bc0434a7aaf6fee0ad1b5af03 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/e9d2b70bc0434a7aaf6fee0ad1b5af03 2024-11-21T03:26:25,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/9a9cc1728b7d406398ffefd68c76e491 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/9a9cc1728b7d406398ffefd68c76e491 2024-11-21T03:26:25,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4dfa975b225b4c14b2a1d46e207cad2a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/4dfa975b225b4c14b2a1d46e207cad2a 2024-11-21T03:26:25,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/99ef0377c98641a981b4a770fea35785 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/99ef0377c98641a981b4a770fea35785 2024-11-21T03:26:25,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/628e2612b04548859ca0e9028eb69788 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/628e2612b04548859ca0e9028eb69788 2024-11-21T03:26:25,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fe26d1d83298489289340a8df6ef2e26 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/fe26d1d83298489289340a8df6ef2e26 2024-11-21T03:26:25,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/96044d9b697f4ccba7c4be3c4d9762b1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/96044d9b697f4ccba7c4be3c4d9762b1 2024-11-21T03:26:25,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a661f59a81f54ac4ae4099ddd895b577 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a661f59a81f54ac4ae4099ddd895b577 2024-11-21T03:26:25,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/0186fc2388af4c0ca83672ac912994c0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/0186fc2388af4c0ca83672ac912994c0 2024-11-21T03:26:25,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a1f70a947a4a404e980e62f2e202de5b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/a1f70a947a4a404e980e62f2e202de5b 2024-11-21T03:26:25,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/494ccebf255c46e791718576a36ec54a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/494ccebf255c46e791718576a36ec54a 2024-11-21T03:26:25,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/566cd56a45b64bd3a69a5c99cfa2cd3f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/566cd56a45b64bd3a69a5c99cfa2cd3f 2024-11-21T03:26:25,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3b5fc7ec4bc844c1b9a7e1be6aa9a316 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/3b5fc7ec4bc844c1b9a7e1be6aa9a316 2024-11-21T03:26:25,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/6779958035d1475887f5068f72f2ed80 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/6779958035d1475887f5068f72f2ed80 2024-11-21T03:26:25,793 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/recovered.edits/497.seqid, newMaxSeqId=497, maxSeqId=1 2024-11-21T03:26:25,793 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. 2024-11-21T03:26:25,794 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 215fc242d64dec572e979318d587db2b: Waiting for close lock at 1732159584960Running coprocessor pre-close hooks at 1732159584960Disabling compacts and flushes for region at 1732159584960Disabling writes for close at 1732159584960Obtaining lock to block concurrent updates at 1732159584960Preparing flush snapshotting stores in 215fc242d64dec572e979318d587db2b at 1732159584960Finished memstore snapshotting TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732159584960Flushing stores of TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b. at 1732159584960Flushing 215fc242d64dec572e979318d587db2b/A: creating writer at 1732159584960Flushing 215fc242d64dec572e979318d587db2b/A: appending metadata at 1732159584963 (+3 ms)Flushing 215fc242d64dec572e979318d587db2b/A: closing flushed file at 1732159584963Flushing 215fc242d64dec572e979318d587db2b/B: creating writer at 1732159584990 (+27 ms)Flushing 215fc242d64dec572e979318d587db2b/B: appending metadata at 1732159584997 (+7 ms)Flushing 215fc242d64dec572e979318d587db2b/B: closing flushed file at 1732159584997Flushing 215fc242d64dec572e979318d587db2b/C: creating writer at 1732159585025 (+28 ms)Flushing 215fc242d64dec572e979318d587db2b/C: appending metadata at 1732159585027 (+2 ms)Flushing 215fc242d64dec572e979318d587db2b/C: closing flushed file at 1732159585028 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68188042: reopening flushed file at 1732159585446 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cccff1a: reopening flushed file at 1732159585454 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2344e5b8: reopening flushed file at 1732159585460 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 215fc242d64dec572e979318d587db2b in 505ms, sequenceid=494, compaction requested=true at 1732159585465 (+5 ms)Writing region close event to WAL at 1732159585788 (+323 ms)Running coprocessor post-close hooks at 1732159585793 (+5 ms)Closed at 1732159585793 2024-11-21T03:26:25,799 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 215fc242d64dec572e979318d587db2b 2024-11-21T03:26:25,799 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=215fc242d64dec572e979318d587db2b, regionState=CLOSED 2024-11-21T03:26:25,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:26:25,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-21T03:26:25,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 215fc242d64dec572e979318d587db2b, server=b69e2a1ee061,45649,1732159441548 in 999 msec 2024-11-21T03:26:25,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-21T03:26:25,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=215fc242d64dec572e979318d587db2b, UNASSIGN in 1.0070 sec 2024-11-21T03:26:25,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-21T03:26:25,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0220 sec 2024-11-21T03:26:25,833 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159585832"}]},"ts":"1732159585832"} 2024-11-21T03:26:25,841 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T03:26:25,841 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T03:26:25,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0500 sec 2024-11-21T03:26:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T03:26:25,941 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:25,949 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T03:26:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:25,964 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:25,966 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T03:26:25,996 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b 2024-11-21T03:26:26,015 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/recovered.edits] 2024-11-21T03:26:26,019 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/499d2726530d4564881e370b1dd6f70d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/499d2726530d4564881e370b1dd6f70d 2024-11-21T03:26:26,020 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/695237b4f2284385856973fb993e579c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/695237b4f2284385856973fb993e579c 2024-11-21T03:26:26,023 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/afcc42c1312940218c927eff70d4a30d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/A/afcc42c1312940218c927eff70d4a30d 2024-11-21T03:26:26,043 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/328d3775356640fda94091d8ace71e9a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/328d3775356640fda94091d8ace71e9a 2024-11-21T03:26:26,050 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/5bc0c3926ffa471e9a7a4c46e18c2f4d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/5bc0c3926ffa471e9a7a4c46e18c2f4d 2024-11-21T03:26:26,054 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e86ab7dbe46441cb8b9dbd93af0105fc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/B/e86ab7dbe46441cb8b9dbd93af0105fc 2024-11-21T03:26:26,067 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/21d7a2c3fd8c4f19857686180b3a94c3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/21d7a2c3fd8c4f19857686180b3a94c3 2024-11-21T03:26:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T03:26:26,083 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/7842b8173a46441688a4f722a25a9c57 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/7842b8173a46441688a4f722a25a9c57 2024-11-21T03:26:26,089 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/928eca51b94c46c3be1f001c71f4a65f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/C/928eca51b94c46c3be1f001c71f4a65f 2024-11-21T03:26:26,092 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/recovered.edits/497.seqid to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b/recovered.edits/497.seqid 2024-11-21T03:26:26,093 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/215fc242d64dec572e979318d587db2b 2024-11-21T03:26:26,093 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T03:26:26,095 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:26,099 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T03:26:26,102 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T03:26:26,103 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:26,104 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T03:26:26,104 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732159586104"}]},"ts":"9223372036854775807"} 2024-11-21T03:26:26,108 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T03:26:26,109 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 215fc242d64dec572e979318d587db2b, NAME => 'TestAcidGuarantees,,1732159560140.215fc242d64dec572e979318d587db2b.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T03:26:26,110 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T03:26:26,110 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732159586110"}]},"ts":"9223372036854775807"} 2024-11-21T03:26:26,123 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T03:26:26,126 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:26,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 177 msec 2024-11-21T03:26:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T03:26:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T03:26:26,297 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:26,327 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239 (was 237) - Thread LEAK? -, OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1048 (was 1045) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2767 (was 1834) - AvailableMemoryMB LEAK? - 2024-11-21T03:26:26,351 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1048, ProcessCount=11, AvailableMemoryMB=2765 2024-11-21T03:26:26,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:26:26,357 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:26:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:26,360 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:26:26,360 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:26,362 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-21T03:26:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-21T03:26:26,363 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:26:26,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742414_1590 (size=959) 2024-11-21T03:26:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-21T03:26:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-21T03:26:26,789 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1 2024-11-21T03:26:26,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742415_1591 (size=53) 2024-11-21T03:26:26,798 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:26:26,798 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0be8e6cfcf7075b5187319303527b737, disabling compactions & flushes 2024-11-21T03:26:26,798 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:26,798 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:26,799 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. after waiting 0 ms 2024-11-21T03:26:26,799 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:26,799 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:26,799 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0be8e6cfcf7075b5187319303527b737: Waiting for close lock at 1732159586798Disabling compacts and flushes for region at 1732159586798Disabling writes for close at 1732159586799 (+1 ms)Writing region close event to WAL at 1732159586799Closed at 1732159586799 2024-11-21T03:26:26,800 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:26:26,800 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732159586800"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732159586800"}]},"ts":"1732159586800"} 2024-11-21T03:26:26,802 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:26:26,803 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:26:26,803 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159586803"}]},"ts":"1732159586803"} 2024-11-21T03:26:26,808 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:26:26,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, ASSIGN}] 2024-11-21T03:26:26,809 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, ASSIGN 2024-11-21T03:26:26,810 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, ASSIGN; state=OFFLINE, location=b69e2a1ee061,45649,1732159441548; forceNewPlan=false, retain=false 2024-11-21T03:26:26,961 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=0be8e6cfcf7075b5187319303527b737, regionState=OPENING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:26,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, ASSIGN because future has completed 2024-11-21T03:26:26,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:26:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-21T03:26:27,118 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:27,119 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:26:27,119 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,119 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:26:27,119 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,119 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,120 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,122 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be8e6cfcf7075b5187319303527b737 columnFamilyName A 2024-11-21T03:26:27,122 DEBUG [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:27,122 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(327): Store=0be8e6cfcf7075b5187319303527b737/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:26:27,123 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,124 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be8e6cfcf7075b5187319303527b737 columnFamilyName B 2024-11-21T03:26:27,124 DEBUG [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:27,124 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(327): Store=0be8e6cfcf7075b5187319303527b737/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:26:27,124 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,126 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be8e6cfcf7075b5187319303527b737 columnFamilyName C 2024-11-21T03:26:27,126 DEBUG [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:27,126 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(327): Store=0be8e6cfcf7075b5187319303527b737/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:26:27,126 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,127 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,127 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,130 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,130 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,130 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:26:27,131 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,133 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:26:27,134 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 0be8e6cfcf7075b5187319303527b737; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59061926, jitterRate=-0.11990872025489807}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:26:27,134 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:27,135 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 0be8e6cfcf7075b5187319303527b737: Running coprocessor pre-open hook at 1732159587119Writing region info on filesystem at 1732159587119Initializing all the Stores at 1732159587120 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159587120Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159587120Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159587120Cleaning up temporary data from old regions at 1732159587130 (+10 ms)Running coprocessor post-open hooks at 1732159587134 (+4 ms)Region opened successfully at 1732159587135 (+1 ms) 2024-11-21T03:26:27,135 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., pid=140, masterSystemTime=1732159587115 2024-11-21T03:26:27,140 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=0be8e6cfcf7075b5187319303527b737, regionState=OPEN, openSeqNum=2, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:27,144 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:27,144 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:27,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:26:27,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-21T03:26:27,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 in 184 msec 2024-11-21T03:26:27,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-21T03:26:27,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, ASSIGN in 340 msec 2024-11-21T03:26:27,164 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:26:27,165 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159587164"}]},"ts":"1732159587164"} 2024-11-21T03:26:27,169 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:26:27,170 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:26:27,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 813 msec 2024-11-21T03:26:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-21T03:26:27,497 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:27,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:27,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:27,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:27,500 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:27,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:27,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:27,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa68098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:27,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:27,502 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e754266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:27,503 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:27,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:27,506 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:26:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is b69e2a1ee061,42023,1732159440464 2024-11-21T03:26:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2b2e39ba 2024-11-21T03:26:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:26:27,508 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:26:27,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:26:27,510 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:26:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:27,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742416_1592 (size=995) 2024-11-21T03:26:27,936 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T03:26:27,936 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T03:26:27,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:26:27,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, REOPEN/MOVE}] 2024-11-21T03:26:27,945 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, REOPEN/MOVE 2024-11-21T03:26:27,946 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0be8e6cfcf7075b5187319303527b737, regionState=CLOSING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:27,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, REOPEN/MOVE because future has completed 2024-11-21T03:26:27,948 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:26:27,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:26:28,108 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,108 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:26:28,108 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 0be8e6cfcf7075b5187319303527b737, disabling compactions & flushes 2024-11-21T03:26:28,108 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:28,109 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:28,109 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. after waiting 0 ms 2024-11-21T03:26:28,109 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:28,120 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T03:26:28,121 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:28,121 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 0be8e6cfcf7075b5187319303527b737: Waiting for close lock at 1732159588108Running coprocessor pre-close hooks at 1732159588108Disabling compacts and flushes for region at 1732159588108Disabling writes for close at 1732159588109 (+1 ms)Writing region close event to WAL at 1732159588114 (+5 ms)Running coprocessor post-close hooks at 1732159588121 (+7 ms)Closed at 1732159588121 2024-11-21T03:26:28,121 WARN [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 0be8e6cfcf7075b5187319303527b737 to self. 2024-11-21T03:26:28,123 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0be8e6cfcf7075b5187319303527b737, regionState=CLOSED 2024-11-21T03:26:28,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:26:28,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-21T03:26:28,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 in 178 msec 2024-11-21T03:26:28,128 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, REOPEN/MOVE; state=CLOSED, location=b69e2a1ee061,45649,1732159441548; forceNewPlan=false, retain=true 2024-11-21T03:26:28,280 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0be8e6cfcf7075b5187319303527b737, regionState=OPENING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, REOPEN/MOVE because future has completed 2024-11-21T03:26:28,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:26:28,438 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:28,438 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:26:28,438 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,439 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:26:28,439 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,439 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,440 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,441 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be8e6cfcf7075b5187319303527b737 columnFamilyName A 2024-11-21T03:26:28,445 DEBUG [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:28,445 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(327): Store=0be8e6cfcf7075b5187319303527b737/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:26:28,446 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,447 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be8e6cfcf7075b5187319303527b737 columnFamilyName B 2024-11-21T03:26:28,447 DEBUG [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:28,448 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(327): Store=0be8e6cfcf7075b5187319303527b737/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:26:28,448 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,449 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be8e6cfcf7075b5187319303527b737 columnFamilyName C 2024-11-21T03:26:28,449 DEBUG [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:28,449 INFO [StoreOpener-0be8e6cfcf7075b5187319303527b737-1 {}] regionserver.HStore(327): Store=0be8e6cfcf7075b5187319303527b737/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:26:28,452 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,453 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,454 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,454 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,455 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,455 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:26:28,456 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,457 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 0be8e6cfcf7075b5187319303527b737; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60270391, jitterRate=-0.10190118849277496}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:26:28,457 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,458 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 0be8e6cfcf7075b5187319303527b737: Running coprocessor pre-open hook at 1732159588439Writing region info on filesystem at 1732159588439Initializing all the Stores at 1732159588439Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159588440 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159588440Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732159588440Cleaning up temporary data from old regions at 1732159588455 (+15 ms)Running coprocessor post-open hooks at 1732159588457 (+2 ms)Region opened successfully at 1732159588457 2024-11-21T03:26:28,458 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., pid=145, masterSystemTime=1732159588435 2024-11-21T03:26:28,460 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0be8e6cfcf7075b5187319303527b737, regionState=OPEN, openSeqNum=5, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:26:28,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-21T03:26:28,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 in 179 msec 2024-11-21T03:26:28,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-21T03:26:28,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, REOPEN/MOVE in 521 msec 2024-11-21T03:26:28,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-21T03:26:28,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-11-21T03:26:28,472 DEBUG [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:28,472 INFO [RS_OPEN_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:28,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 962 msec 2024-11-21T03:26:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T03:26:28,476 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:28,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:28,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:28,479 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52cdaf4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:28,481 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31d980ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:28,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:28,483 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d84d1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:28,486 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:28,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@474bd4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:28,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:28,489 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@118e6510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:28,491 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa6c2b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:28,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:28,498 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739b9a7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:28,499 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c85b8fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:28,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:28,506 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123f00a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:28,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:28,508 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:28,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46324e4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:28,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:28,513 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349c1ac7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:28,515 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67bc3a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:28,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:28,517 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e27f8f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:28,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:28,519 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:28,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be7b604, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,521 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:28,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:28,522 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173f3f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:28,524 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c6fce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:28,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:28,532 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b3f4a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:28,534 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce486e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b69e2a1ee061,42023,-1 for getting cluster id 2024-11-21T03:26:28,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:26:28,536 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'acaff57a-38ea-4fc8-a8f9-17336e0f736f' 2024-11-21T03:26:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:26:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "acaff57a-38ea-4fc8-a8f9-17336e0f736f" 2024-11-21T03:26:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe6315c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b69e2a1ee061,42023,-1] 2024-11-21T03:26:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:26:28,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:28,539 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:26:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75bc91ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:26:28,552 DEBUG [Thread-2625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:28,552 DEBUG [Thread-2632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:28,553 DEBUG [Thread-2636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:28,553 DEBUG [Thread-2617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:28,554 DEBUG [Thread-2623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:28,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:28,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:28,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:28,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:28,556 DEBUG [Thread-2619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:28,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:28,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:26:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:28,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:28,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:28,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:26:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:26:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:26:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:26:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:26:28,565 DEBUG [Thread-2628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:28,566 DEBUG [Thread-2634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:28,567 DEBUG [Thread-2621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:28,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:28,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:28,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:28,568 DEBUG [Thread-2621 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:28,568 DEBUG [Thread-2630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:26:28,568 DEBUG [Thread-2621 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:28,569 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:28,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:28,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b69e2a1ee061,45649,1732159441548, seqNum=-1] 2024-11-21T03:26:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:26:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:26:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-21T03:26:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:26:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:28,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:26:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T03:26:28,572 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:26:28,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:26:28,574 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:28,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5] 2024-11-21T03:26:28,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159648609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159648611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159648610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159648615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159648617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215875d6b76bdb48d88550b6f588445780_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159588567/Put/seqid=0 2024-11-21T03:26:28,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742417_1593 (size=12154) 2024-11-21T03:26:28,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:28,658 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215875d6b76bdb48d88550b6f588445780_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215875d6b76bdb48d88550b6f588445780_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,660 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2d3ea8302c1047918532df45ce8e56c3, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:28,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2d3ea8302c1047918532df45ce8e56c3 is 175, key is test_row_0/A:col10/1732159588567/Put/seqid=0 2024-11-21T03:26:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T03:26:28,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742418_1594 (size=30955) 2024-11-21T03:26:28,702 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2d3ea8302c1047918532df45ce8e56c3 2024-11-21T03:26:28,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-21T03:26:28,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:28,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:28,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159648728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159648728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159648729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159648729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159648729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/e82d707d72bb430c84e12b981cb05cd6 is 50, key is test_row_0/B:col10/1732159588567/Put/seqid=0 2024-11-21T03:26:28,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742419_1595 (size=12001) 2024-11-21T03:26:28,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/e82d707d72bb430c84e12b981cb05cd6 2024-11-21T03:26:28,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d1c5b2138e854f6c815414d61d51675c is 50, key is test_row_0/C:col10/1732159588567/Put/seqid=0 2024-11-21T03:26:28,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742420_1596 (size=12001) 2024-11-21T03:26:28,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d1c5b2138e854f6c815414d61d51675c 2024-11-21T03:26:28,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2d3ea8302c1047918532df45ce8e56c3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2d3ea8302c1047918532df45ce8e56c3 2024-11-21T03:26:28,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2d3ea8302c1047918532df45ce8e56c3, entries=150, sequenceid=16, filesize=30.2 K 2024-11-21T03:26:28,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/e82d707d72bb430c84e12b981cb05cd6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/e82d707d72bb430c84e12b981cb05cd6 2024-11-21T03:26:28,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/e82d707d72bb430c84e12b981cb05cd6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-21T03:26:28,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d1c5b2138e854f6c815414d61d51675c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1c5b2138e854f6c815414d61d51675c 2024-11-21T03:26:28,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1c5b2138e854f6c815414d61d51675c, entries=150, sequenceid=16, filesize=11.7 K 2024-11-21T03:26:28,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0be8e6cfcf7075b5187319303527b737 in 285ms, sequenceid=16, compaction requested=false 2024-11-21T03:26:28,856 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T03:26:28,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:28,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-21T03:26:28,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:28,883 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:26:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T03:26:28,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121986eead2503949ef8b61eae4133b9585_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159588615/Put/seqid=0 2024-11-21T03:26:28,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742421_1597 (size=12154) 2024-11-21T03:26:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:28,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:28,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159648981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159648982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159648982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159648983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159648984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159649098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159649098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159649099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159649100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159649100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T03:26:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159649308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159649308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159649309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159649310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159649310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,318 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T03:26:29,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:29,359 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121986eead2503949ef8b61eae4133b9585_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121986eead2503949ef8b61eae4133b9585_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/5fb13d7c49ca47f5b4ebecad01633e86, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/5fb13d7c49ca47f5b4ebecad01633e86 is 175, key is test_row_0/A:col10/1732159588615/Put/seqid=0 2024-11-21T03:26:29,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742422_1598 (size=30955) 2024-11-21T03:26:29,376 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/5fb13d7c49ca47f5b4ebecad01633e86 2024-11-21T03:26:29,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/0fbf87bdf07544c2ab1bf0e78eed1abf is 50, key is test_row_0/B:col10/1732159588615/Put/seqid=0 2024-11-21T03:26:29,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742423_1599 (size=12001) 2024-11-21T03:26:29,436 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/0fbf87bdf07544c2ab1bf0e78eed1abf 2024-11-21T03:26:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d1e6751ba28343a6ba41e7bdbe7ae085 is 50, key is test_row_0/C:col10/1732159588615/Put/seqid=0 2024-11-21T03:26:29,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742424_1600 (size=12001) 2024-11-21T03:26:29,505 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d1e6751ba28343a6ba41e7bdbe7ae085 2024-11-21T03:26:29,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/5fb13d7c49ca47f5b4ebecad01633e86 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5fb13d7c49ca47f5b4ebecad01633e86 2024-11-21T03:26:29,515 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5fb13d7c49ca47f5b4ebecad01633e86, entries=150, sequenceid=40, filesize=30.2 K 2024-11-21T03:26:29,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/0fbf87bdf07544c2ab1bf0e78eed1abf as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/0fbf87bdf07544c2ab1bf0e78eed1abf 2024-11-21T03:26:29,525 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/0fbf87bdf07544c2ab1bf0e78eed1abf, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T03:26:29,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d1e6751ba28343a6ba41e7bdbe7ae085 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1e6751ba28343a6ba41e7bdbe7ae085 2024-11-21T03:26:29,530 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1e6751ba28343a6ba41e7bdbe7ae085, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T03:26:29,531 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 0be8e6cfcf7075b5187319303527b737 in 648ms, sequenceid=40, compaction requested=false 2024-11-21T03:26:29,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:29,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:29,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-21T03:26:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-21T03:26:29,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-21T03:26:29,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 959 msec 2024-11-21T03:26:29,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 969 msec 2024-11-21T03:26:29,623 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:29,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:26:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:29,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f3b1a0c023cf4091b739b5eae9cc5f4d_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159589622/Put/seqid=0 2024-11-21T03:26:29,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159649652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159649653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159649662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159649663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159649664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742425_1601 (size=12154) 2024-11-21T03:26:29,685 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:29,688 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f3b1a0c023cf4091b739b5eae9cc5f4d_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f3b1a0c023cf4091b739b5eae9cc5f4d_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:29,689 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/317d4444d8674a8aaa1ac353f338303b, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:29,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/317d4444d8674a8aaa1ac353f338303b is 175, key is test_row_0/A:col10/1732159589622/Put/seqid=0 2024-11-21T03:26:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T03:26:29,708 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:29,708 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-21T03:26:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T03:26:29,712 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:29,713 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:29,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:29,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742426_1602 (size=30955) 2024-11-21T03:26:29,732 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/317d4444d8674a8aaa1ac353f338303b 2024-11-21T03:26:29,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/bbeac2fac4f745a1813f7bbb6066781b is 50, key is test_row_0/B:col10/1732159589622/Put/seqid=0 2024-11-21T03:26:29,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742427_1603 (size=12001) 2024-11-21T03:26:29,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/bbeac2fac4f745a1813f7bbb6066781b 2024-11-21T03:26:29,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159649778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159649778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159649778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159649784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159649784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:29,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d4dbb0911a464ebfa38d69947e395cf4 is 50, key is test_row_0/C:col10/1732159589622/Put/seqid=0 2024-11-21T03:26:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T03:26:29,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742428_1604 (size=12001) 2024-11-21T03:26:29,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d4dbb0911a464ebfa38d69947e395cf4 2024-11-21T03:26:29,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/317d4444d8674a8aaa1ac353f338303b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/317d4444d8674a8aaa1ac353f338303b 2024-11-21T03:26:29,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/317d4444d8674a8aaa1ac353f338303b, entries=150, sequenceid=55, filesize=30.2 K 2024-11-21T03:26:29,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/bbeac2fac4f745a1813f7bbb6066781b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/bbeac2fac4f745a1813f7bbb6066781b 2024-11-21T03:26:29,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T03:26:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:29,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:29,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:29,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/bbeac2fac4f745a1813f7bbb6066781b, entries=150, sequenceid=55, filesize=11.7 K 2024-11-21T03:26:29,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d4dbb0911a464ebfa38d69947e395cf4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d4dbb0911a464ebfa38d69947e395cf4 2024-11-21T03:26:29,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d4dbb0911a464ebfa38d69947e395cf4, entries=150, sequenceid=55, filesize=11.7 K 2024-11-21T03:26:29,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0be8e6cfcf7075b5187319303527b737 in 318ms, sequenceid=55, compaction requested=true 2024-11-21T03:26:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:29,941 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:29,942 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:29,942 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/A is initiating minor compaction (all files) 2024-11-21T03:26:29,942 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/A in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:29,942 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2d3ea8302c1047918532df45ce8e56c3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5fb13d7c49ca47f5b4ebecad01633e86, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/317d4444d8674a8aaa1ac353f338303b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=90.7 K 2024-11-21T03:26:29,943 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:29,943 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2d3ea8302c1047918532df45ce8e56c3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5fb13d7c49ca47f5b4ebecad01633e86, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/317d4444d8674a8aaa1ac353f338303b] 2024-11-21T03:26:29,943 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:29,944 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d3ea8302c1047918532df45ce8e56c3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732159588566 2024-11-21T03:26:29,945 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fb13d7c49ca47f5b4ebecad01633e86, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732159588608 2024-11-21T03:26:29,945 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 317d4444d8674a8aaa1ac353f338303b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732159588982 2024-11-21T03:26:29,946 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:29,946 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/B is initiating minor compaction (all files) 2024-11-21T03:26:29,947 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/B in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:29,947 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/e82d707d72bb430c84e12b981cb05cd6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/0fbf87bdf07544c2ab1bf0e78eed1abf, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/bbeac2fac4f745a1813f7bbb6066781b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=35.2 K 2024-11-21T03:26:29,948 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e82d707d72bb430c84e12b981cb05cd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732159588566 2024-11-21T03:26:29,948 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fbf87bdf07544c2ab1bf0e78eed1abf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732159588608 2024-11-21T03:26:29,949 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting bbeac2fac4f745a1813f7bbb6066781b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732159588982 2024-11-21T03:26:29,958 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:29,963 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#B#compaction#520 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:29,964 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/6bfd16b893e14be8bca290dbe004a087 is 50, key is test_row_0/B:col10/1732159589622/Put/seqid=0 2024-11-21T03:26:29,969 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411212ec5947915d244ecb2d1793c4aed8191_0be8e6cfcf7075b5187319303527b737 store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:29,971 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411212ec5947915d244ecb2d1793c4aed8191_0be8e6cfcf7075b5187319303527b737, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:29,971 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212ec5947915d244ecb2d1793c4aed8191_0be8e6cfcf7075b5187319303527b737 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:29,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T03:26:29,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742429_1605 (size=12104) 2024-11-21T03:26:30,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a856c9db6f1544bb9c7f1a7d86f273ff_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159589649/Put/seqid=0 2024-11-21T03:26:30,027 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/6bfd16b893e14be8bca290dbe004a087 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/6bfd16b893e14be8bca290dbe004a087 2024-11-21T03:26:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T03:26:30,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159650017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159650020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159650021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742430_1606 (size=4469) 2024-11-21T03:26:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,033 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/B of 0be8e6cfcf7075b5187319303527b737 into 6bfd16b893e14be8bca290dbe004a087(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:30,034 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:30,034 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/B, priority=13, startTime=1732159589941; duration=0sec 2024-11-21T03:26:30,034 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:30,034 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:B 2024-11-21T03:26:30,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742431_1607 (size=17034) 2024-11-21T03:26:30,034 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:30,034 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#A#compaction#519 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:30,035 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/77be856bfd0d4d4e8740ff55476cd0d7 is 175, key is test_row_0/A:col10/1732159589622/Put/seqid=0 2024-11-21T03:26:30,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T03:26:30,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159650025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:30,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159650032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,039 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:30,040 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/C is initiating minor compaction (all files) 2024-11-21T03:26:30,040 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/C in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,040 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1c5b2138e854f6c815414d61d51675c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1e6751ba28343a6ba41e7bdbe7ae085, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d4dbb0911a464ebfa38d69947e395cf4] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=35.2 K 2024-11-21T03:26:30,040 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c5b2138e854f6c815414d61d51675c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732159588566 2024-11-21T03:26:30,041 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e6751ba28343a6ba41e7bdbe7ae085, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732159588608 2024-11-21T03:26:30,041 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d4dbb0911a464ebfa38d69947e395cf4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732159588982 2024-11-21T03:26:30,051 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a856c9db6f1544bb9c7f1a7d86f273ff_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a856c9db6f1544bb9c7f1a7d86f273ff_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:30,052 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/62e5b970271342d7adee3f24609b633c, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:30,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/62e5b970271342d7adee3f24609b633c is 175, key is test_row_0/A:col10/1732159589649/Put/seqid=0 2024-11-21T03:26:30,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742432_1608 (size=31058) 2024-11-21T03:26:30,059 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#C#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:30,060 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/ed21f60fa49046dea058a8c611c5b7c6 is 50, key is test_row_0/C:col10/1732159589622/Put/seqid=0 2024-11-21T03:26:30,069 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/77be856bfd0d4d4e8740ff55476cd0d7 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/77be856bfd0d4d4e8740ff55476cd0d7 2024-11-21T03:26:30,072 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/A of 0be8e6cfcf7075b5187319303527b737 into 77be856bfd0d4d4e8740ff55476cd0d7(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:30,072 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:30,072 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/A, priority=13, startTime=1732159589941; duration=0sec 2024-11-21T03:26:30,072 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:30,072 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:A 2024-11-21T03:26:30,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742433_1609 (size=48139) 2024-11-21T03:26:30,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742434_1610 (size=12104) 2024-11-21T03:26:30,116 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/ed21f60fa49046dea058a8c611c5b7c6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/ed21f60fa49046dea058a8c611c5b7c6 2024-11-21T03:26:30,124 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/C of 0be8e6cfcf7075b5187319303527b737 into ed21f60fa49046dea058a8c611c5b7c6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:30,125 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:30,125 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/C, priority=13, startTime=1732159589941; duration=0sec 2024-11-21T03:26:30,125 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:30,125 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:C 2024-11-21T03:26:30,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159650140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159650141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159650141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159650148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159650148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T03:26:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T03:26:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T03:26:30,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159650352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159650353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159650354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159650358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159650358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,490 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/62e5b970271342d7adee3f24609b633c 2024-11-21T03:26:30,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T03:26:30,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:30,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/9c31ea85543c4823995b933c8a130c7f is 50, key is test_row_0/B:col10/1732159589649/Put/seqid=0 2024-11-21T03:26:30,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742435_1611 (size=12001) 2024-11-21T03:26:30,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/9c31ea85543c4823995b933c8a130c7f 2024-11-21T03:26:30,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/a61665f997b346fc9a503fbfef6e858f is 50, key is test_row_0/C:col10/1732159589649/Put/seqid=0 2024-11-21T03:26:30,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T03:26:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:30,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742436_1612 (size=12001) 2024-11-21T03:26:30,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/a61665f997b346fc9a503fbfef6e858f 2024-11-21T03:26:30,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/62e5b970271342d7adee3f24609b633c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/62e5b970271342d7adee3f24609b633c 2024-11-21T03:26:30,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159650669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159650669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159650669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159650669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159650672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:30,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:30,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/62e5b970271342d7adee3f24609b633c, entries=250, sequenceid=78, filesize=47.0 K 2024-11-21T03:26:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/9c31ea85543c4823995b933c8a130c7f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/9c31ea85543c4823995b933c8a130c7f 2024-11-21T03:26:30,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/9c31ea85543c4823995b933c8a130c7f, entries=150, sequenceid=78, filesize=11.7 K 2024-11-21T03:26:30,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/a61665f997b346fc9a503fbfef6e858f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/a61665f997b346fc9a503fbfef6e858f 2024-11-21T03:26:30,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/a61665f997b346fc9a503fbfef6e858f, entries=150, sequenceid=78, filesize=11.7 K 2024-11-21T03:26:30,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 0be8e6cfcf7075b5187319303527b737 in 748ms, sequenceid=78, compaction requested=false 2024-11-21T03:26:30,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:30,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-21T03:26:30,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:30,808 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:26:30,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112106e13a0304424b13b1245ebdf6e9f97b_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159590023/Put/seqid=0 2024-11-21T03:26:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T03:26:30,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742437_1613 (size=12154) 2024-11-21T03:26:30,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:30,899 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112106e13a0304424b13b1245ebdf6e9f97b_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112106e13a0304424b13b1245ebdf6e9f97b_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:30,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0356ebdd74c1486397a6baf071b2c2a8, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:30,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0356ebdd74c1486397a6baf071b2c2a8 is 175, key is test_row_0/A:col10/1732159590023/Put/seqid=0 2024-11-21T03:26:30,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742438_1614 (size=30955) 2024-11-21T03:26:31,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:31,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159651253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159651253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159651270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159651274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159651275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,334 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:26:31,366 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0356ebdd74c1486397a6baf071b2c2a8 2024-11-21T03:26:31,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/4c4dfd105d2940679329d6b2f4e13bad is 50, key is test_row_0/B:col10/1732159590023/Put/seqid=0 2024-11-21T03:26:31,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159651387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159651388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159651388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159651388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159651388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742439_1615 (size=12001) 2024-11-21T03:26:31,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159651597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159651597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159651597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159651598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159651598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,824 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/4c4dfd105d2940679329d6b2f4e13bad 2024-11-21T03:26:31,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/64deb3ab9bef4d0b81e771a9f60f18f8 is 50, key is test_row_0/C:col10/1732159590023/Put/seqid=0 2024-11-21T03:26:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T03:26:31,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742440_1616 (size=12001) 2024-11-21T03:26:31,877 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/64deb3ab9bef4d0b81e771a9f60f18f8 2024-11-21T03:26:31,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0356ebdd74c1486397a6baf071b2c2a8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0356ebdd74c1486397a6baf071b2c2a8 2024-11-21T03:26:31,889 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0356ebdd74c1486397a6baf071b2c2a8, entries=150, sequenceid=95, filesize=30.2 K 2024-11-21T03:26:31,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/4c4dfd105d2940679329d6b2f4e13bad as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4c4dfd105d2940679329d6b2f4e13bad 2024-11-21T03:26:31,896 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4c4dfd105d2940679329d6b2f4e13bad, entries=150, sequenceid=95, filesize=11.7 K 2024-11-21T03:26:31,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/64deb3ab9bef4d0b81e771a9f60f18f8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/64deb3ab9bef4d0b81e771a9f60f18f8 2024-11-21T03:26:31,903 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/64deb3ab9bef4d0b81e771a9f60f18f8, entries=150, sequenceid=95, filesize=11.7 K 2024-11-21T03:26:31,906 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 0be8e6cfcf7075b5187319303527b737 in 1098ms, sequenceid=95, compaction requested=true 2024-11-21T03:26:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:31,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-21T03:26:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-21T03:26:31,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-21T03:26:31,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1960 sec 2024-11-21T03:26:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:31,914 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:31,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:26:31,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.2120 sec 2024-11-21T03:26:31,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121124c4dc16df9419f83e988c9b68b260e_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159591250/Put/seqid=0 2024-11-21T03:26:31,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742441_1617 (size=12154) 2024-11-21T03:26:31,937 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:31,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121124c4dc16df9419f83e988c9b68b260e_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121124c4dc16df9419f83e988c9b68b260e_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:31,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159651931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,946 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0c3070fb25d84a86bce7abc94dbb9b9b, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:31,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0c3070fb25d84a86bce7abc94dbb9b9b is 175, key is test_row_0/A:col10/1732159591250/Put/seqid=0 2024-11-21T03:26:31,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159651933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159651943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159651943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:31,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159651944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:31,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:31,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:31,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742442_1618 (size=30955) 2024-11-21T03:26:32,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159652048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159652058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159652058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159652058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159652058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159652257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159652268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159652268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159652268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159652269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,380 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0c3070fb25d84a86bce7abc94dbb9b9b 2024-11-21T03:26:32,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/79eef470121e4eea9da5c3c31551071f is 50, key is test_row_0/B:col10/1732159591250/Put/seqid=0 2024-11-21T03:26:32,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742443_1619 (size=12001) 2024-11-21T03:26:32,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/79eef470121e4eea9da5c3c31551071f 2024-11-21T03:26:32,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c0acab149f16493597ec0cae6d0cdf44 is 50, key is test_row_0/C:col10/1732159591250/Put/seqid=0 2024-11-21T03:26:32,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742444_1620 (size=12001) 2024-11-21T03:26:32,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c0acab149f16493597ec0cae6d0cdf44 2024-11-21T03:26:32,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0c3070fb25d84a86bce7abc94dbb9b9b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0c3070fb25d84a86bce7abc94dbb9b9b 2024-11-21T03:26:32,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0c3070fb25d84a86bce7abc94dbb9b9b, entries=150, sequenceid=119, filesize=30.2 K 2024-11-21T03:26:32,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/79eef470121e4eea9da5c3c31551071f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/79eef470121e4eea9da5c3c31551071f 2024-11-21T03:26:32,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/79eef470121e4eea9da5c3c31551071f, entries=150, sequenceid=119, filesize=11.7 K 2024-11-21T03:26:32,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c0acab149f16493597ec0cae6d0cdf44 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c0acab149f16493597ec0cae6d0cdf44 2024-11-21T03:26:32,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c0acab149f16493597ec0cae6d0cdf44, entries=150, sequenceid=119, filesize=11.7 K 2024-11-21T03:26:32,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 0be8e6cfcf7075b5187319303527b737 in 613ms, sequenceid=119, compaction requested=true 2024-11-21T03:26:32,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:32,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:32,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:32,528 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:26:32,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:32,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:32,528 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:26:32,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:32,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:32,529 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:26:32,529 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:26:32,529 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/A is initiating minor compaction (all files) 2024-11-21T03:26:32,529 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/B is initiating minor compaction (all files) 2024-11-21T03:26:32,529 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/B in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:32,529 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/A in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:32,529 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/6bfd16b893e14be8bca290dbe004a087, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/9c31ea85543c4823995b933c8a130c7f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4c4dfd105d2940679329d6b2f4e13bad, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/79eef470121e4eea9da5c3c31551071f] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=47.0 K 2024-11-21T03:26:32,529 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/77be856bfd0d4d4e8740ff55476cd0d7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/62e5b970271342d7adee3f24609b633c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0356ebdd74c1486397a6baf071b2c2a8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0c3070fb25d84a86bce7abc94dbb9b9b] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=137.8 K 2024-11-21T03:26:32,529 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:32,529 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/77be856bfd0d4d4e8740ff55476cd0d7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/62e5b970271342d7adee3f24609b633c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0356ebdd74c1486397a6baf071b2c2a8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0c3070fb25d84a86bce7abc94dbb9b9b] 2024-11-21T03:26:32,530 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bfd16b893e14be8bca290dbe004a087, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732159588982 2024-11-21T03:26:32,530 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c31ea85543c4823995b933c8a130c7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732159589649 2024-11-21T03:26:32,530 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77be856bfd0d4d4e8740ff55476cd0d7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732159588982 2024-11-21T03:26:32,531 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62e5b970271342d7adee3f24609b633c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732159589649 2024-11-21T03:26:32,531 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c4dfd105d2940679329d6b2f4e13bad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732159590015 2024-11-21T03:26:32,531 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 79eef470121e4eea9da5c3c31551071f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732159591250 2024-11-21T03:26:32,531 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0356ebdd74c1486397a6baf071b2c2a8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732159590015 2024-11-21T03:26:32,531 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c3070fb25d84a86bce7abc94dbb9b9b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732159591250 2024-11-21T03:26:32,550 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#B#compaction#531 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:32,551 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/5c078e2ff55d4b3eb5f8cdea443bb384 is 50, key is test_row_0/B:col10/1732159591250/Put/seqid=0 2024-11-21T03:26:32,557 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:32,572 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:32,573 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411210ec23e92334a44b1a67872044223f5f6_0be8e6cfcf7075b5187319303527b737 store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:32,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:26:32,575 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411210ec23e92334a44b1a67872044223f5f6_0be8e6cfcf7075b5187319303527b737, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:32,576 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210ec23e92334a44b1a67872044223f5f6_0be8e6cfcf7075b5187319303527b737 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:32,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742445_1621 (size=12241) 2024-11-21T03:26:32,623 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/5c078e2ff55d4b3eb5f8cdea443bb384 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5c078e2ff55d4b3eb5f8cdea443bb384 2024-11-21T03:26:32,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112172e6ab479f184a7a9376351f616560a5_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159591935/Put/seqid=0 2024-11-21T03:26:32,629 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/B of 0be8e6cfcf7075b5187319303527b737 into 5c078e2ff55d4b3eb5f8cdea443bb384(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:32,629 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:32,629 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/B, priority=12, startTime=1732159592528; duration=0sec 2024-11-21T03:26:32,629 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:32,629 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:B 2024-11-21T03:26:32,629 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:26:32,630 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:26:32,630 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/C is initiating minor compaction (all files) 2024-11-21T03:26:32,630 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/C in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:32,631 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/ed21f60fa49046dea058a8c611c5b7c6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/a61665f997b346fc9a503fbfef6e858f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/64deb3ab9bef4d0b81e771a9f60f18f8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c0acab149f16493597ec0cae6d0cdf44] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=47.0 K 2024-11-21T03:26:32,631 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting ed21f60fa49046dea058a8c611c5b7c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732159588982 2024-11-21T03:26:32,631 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting a61665f997b346fc9a503fbfef6e858f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732159589649 2024-11-21T03:26:32,631 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 64deb3ab9bef4d0b81e771a9f60f18f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732159590015 2024-11-21T03:26:32,632 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting c0acab149f16493597ec0cae6d0cdf44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732159591250 2024-11-21T03:26:32,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742446_1622 (size=4469) 2024-11-21T03:26:32,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159652633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159652635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,652 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#A#compaction#532 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,652 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/f235a001d513492eb879b6d59a695f7c is 175, key is test_row_0/A:col10/1732159591250/Put/seqid=0 2024-11-21T03:26:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159652649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159652650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159652651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742447_1623 (size=12254) 2024-11-21T03:26:32,676 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#C#compaction#534 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:32,676 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/6416548be1af438b86b9ec4ccca94247 is 50, key is test_row_0/C:col10/1732159591250/Put/seqid=0 2024-11-21T03:26:32,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742448_1624 (size=31195) 2024-11-21T03:26:32,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742449_1625 (size=12241) 2024-11-21T03:26:32,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159652758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159652758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159652767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159652768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159652768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159652969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159652977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159652977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159652988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159652988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,070 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:33,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112172e6ab479f184a7a9376351f616560a5_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112172e6ab479f184a7a9376351f616560a5_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:33,089 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/bb51372614c6405787e90b784727b755, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:33,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/bb51372614c6405787e90b784727b755 is 175, key is test_row_0/A:col10/1732159591935/Put/seqid=0 2024-11-21T03:26:33,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742450_1626 (size=31055) 2024-11-21T03:26:33,132 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/f235a001d513492eb879b6d59a695f7c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f235a001d513492eb879b6d59a695f7c 2024-11-21T03:26:33,148 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/A of 0be8e6cfcf7075b5187319303527b737 into f235a001d513492eb879b6d59a695f7c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:33,148 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:33,148 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/A, priority=12, startTime=1732159592528; duration=0sec 2024-11-21T03:26:33,148 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:33,149 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:A 2024-11-21T03:26:33,158 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/6416548be1af438b86b9ec4ccca94247 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/6416548be1af438b86b9ec4ccca94247 2024-11-21T03:26:33,172 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/C of 0be8e6cfcf7075b5187319303527b737 into 6416548be1af438b86b9ec4ccca94247(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:33,172 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:33,172 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/C, priority=12, startTime=1732159592528; duration=0sec 2024-11-21T03:26:33,172 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:33,172 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:C 2024-11-21T03:26:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159653289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159653290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159653290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159653298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159653298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,516 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/bb51372614c6405787e90b784727b755 2024-11-21T03:26:33,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/730ad1d1ee8244b581037380d72f702c is 50, key is test_row_0/B:col10/1732159591935/Put/seqid=0 2024-11-21T03:26:33,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742451_1627 (size=12101) 2024-11-21T03:26:33,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/730ad1d1ee8244b581037380d72f702c 2024-11-21T03:26:33,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/0f10b0f4fc6948d0b86de5327bf530bc is 50, key is test_row_0/C:col10/1732159591935/Put/seqid=0 2024-11-21T03:26:33,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742452_1628 (size=12101) 2024-11-21T03:26:33,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/0f10b0f4fc6948d0b86de5327bf530bc 2024-11-21T03:26:33,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/bb51372614c6405787e90b784727b755 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/bb51372614c6405787e90b784727b755 2024-11-21T03:26:33,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/bb51372614c6405787e90b784727b755, entries=150, sequenceid=133, filesize=30.3 K 2024-11-21T03:26:33,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/730ad1d1ee8244b581037380d72f702c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/730ad1d1ee8244b581037380d72f702c 2024-11-21T03:26:33,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/730ad1d1ee8244b581037380d72f702c, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T03:26:33,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/0f10b0f4fc6948d0b86de5327bf530bc as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0f10b0f4fc6948d0b86de5327bf530bc 2024-11-21T03:26:33,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0f10b0f4fc6948d0b86de5327bf530bc, entries=150, sequenceid=133, filesize=11.8 K 2024-11-21T03:26:33,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 0be8e6cfcf7075b5187319303527b737 in 1008ms, sequenceid=133, compaction requested=false 2024-11-21T03:26:33,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:33,799 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:33,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:26:33,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216c630cffaab442f4bf9c08c3c0f1e390_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159593798/Put/seqid=0 2024-11-21T03:26:33,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742453_1629 (size=14794) 2024-11-21T03:26:33,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159653812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159653813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159653813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159653813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159653817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-21T03:26:33,867 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:33,868 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-21T03:26:33,871 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:33,871 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:33,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T03:26:33,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159653928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159653928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159653928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159653928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159653928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T03:26:34,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T03:26:34,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:34,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159654137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159654137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159654138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159654138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159654140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T03:26:34,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:34,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T03:26:34,224 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:34,231 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216c630cffaab442f4bf9c08c3c0f1e390_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216c630cffaab442f4bf9c08c3c0f1e390_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:34,231 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/a32feca658d64afbbb9a00c6e20ef142, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:34,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/a32feca658d64afbbb9a00c6e20ef142 is 175, key is test_row_0/A:col10/1732159593798/Put/seqid=0 2024-11-21T03:26:34,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742454_1630 (size=39749) 2024-11-21T03:26:34,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T03:26:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159654448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159654448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159654453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159654453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159654453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T03:26:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T03:26:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T03:26:34,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/a32feca658d64afbbb9a00c6e20ef142 2024-11-21T03:26:34,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/95e523ac0c3548499723c3b3543cd479 is 50, key is test_row_0/B:col10/1732159593798/Put/seqid=0 2024-11-21T03:26:34,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742455_1631 (size=12151) 2024-11-21T03:26:34,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/95e523ac0c3548499723c3b3543cd479 2024-11-21T03:26:34,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d1677cf8b98049b3ae94a9ce443ad8ce is 50, key is test_row_0/C:col10/1732159593798/Put/seqid=0 2024-11-21T03:26:34,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742456_1632 (size=12151) 2024-11-21T03:26:34,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T03:26:34,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:34,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159654958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159654958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159654960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159654967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159654967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:34,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T03:26:34,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:34,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T03:26:35,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T03:26:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:35,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:35,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d1677cf8b98049b3ae94a9ce443ad8ce 2024-11-21T03:26:35,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/a32feca658d64afbbb9a00c6e20ef142 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/a32feca658d64afbbb9a00c6e20ef142 2024-11-21T03:26:35,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/a32feca658d64afbbb9a00c6e20ef142, entries=200, sequenceid=160, filesize=38.8 K 2024-11-21T03:26:35,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/95e523ac0c3548499723c3b3543cd479 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/95e523ac0c3548499723c3b3543cd479 2024-11-21T03:26:35,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/95e523ac0c3548499723c3b3543cd479, entries=150, sequenceid=160, filesize=11.9 K 2024-11-21T03:26:35,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/d1677cf8b98049b3ae94a9ce443ad8ce as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1677cf8b98049b3ae94a9ce443ad8ce 2024-11-21T03:26:35,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1677cf8b98049b3ae94a9ce443ad8ce, entries=150, sequenceid=160, filesize=11.9 K 2024-11-21T03:26:35,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 0be8e6cfcf7075b5187319303527b737 in 1437ms, sequenceid=160, compaction requested=true 2024-11-21T03:26:35,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:35,237 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:35,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:35,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:35,237 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:35,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:35,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:35,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:35,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:35,245 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:35,245 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/B is initiating minor compaction (all files) 2024-11-21T03:26:35,245 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/B in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:35,245 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5c078e2ff55d4b3eb5f8cdea443bb384, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/730ad1d1ee8244b581037380d72f702c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/95e523ac0c3548499723c3b3543cd479] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=35.6 K 2024-11-21T03:26:35,245 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:35,246 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/A is initiating minor compaction (all files) 2024-11-21T03:26:35,246 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/A in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:35,246 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f235a001d513492eb879b6d59a695f7c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/bb51372614c6405787e90b784727b755, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/a32feca658d64afbbb9a00c6e20ef142] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=99.6 K 2024-11-21T03:26:35,246 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:35,246 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f235a001d513492eb879b6d59a695f7c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/bb51372614c6405787e90b784727b755, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/a32feca658d64afbbb9a00c6e20ef142] 2024-11-21T03:26:35,246 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c078e2ff55d4b3eb5f8cdea443bb384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732159591250 2024-11-21T03:26:35,247 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f235a001d513492eb879b6d59a695f7c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732159591250 2024-11-21T03:26:35,247 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 730ad1d1ee8244b581037380d72f702c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159591931 2024-11-21T03:26:35,247 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e523ac0c3548499723c3b3543cd479, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732159592649 2024-11-21T03:26:35,247 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb51372614c6405787e90b784727b755, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159591931 2024-11-21T03:26:35,248 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting a32feca658d64afbbb9a00c6e20ef142, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732159592624 2024-11-21T03:26:35,256 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:35,266 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#B#compaction#541 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:35,266 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/d0891e355ec0432eb46fa134f35d553f is 50, key is test_row_0/B:col10/1732159593798/Put/seqid=0 2024-11-21T03:26:35,269 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121a44fd0a827f740b2bd042c3218e8d445_0be8e6cfcf7075b5187319303527b737 store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:35,271 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121a44fd0a827f740b2bd042c3218e8d445_0be8e6cfcf7075b5187319303527b737, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:35,271 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a44fd0a827f740b2bd042c3218e8d445_0be8e6cfcf7075b5187319303527b737 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:35,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-21T03:26:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:35,290 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:26:35,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742457_1633 (size=4469) 2024-11-21T03:26:35,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742458_1634 (size=12493) 2024-11-21T03:26:35,324 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/d0891e355ec0432eb46fa134f35d553f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/d0891e355ec0432eb46fa134f35d553f 2024-11-21T03:26:35,328 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/B of 0be8e6cfcf7075b5187319303527b737 into d0891e355ec0432eb46fa134f35d553f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:35,328 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:35,328 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/B, priority=13, startTime=1732159595237; duration=0sec 2024-11-21T03:26:35,328 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:35,328 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:B 2024-11-21T03:26:35,328 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:35,329 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:35,329 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/C is initiating minor compaction (all files) 2024-11-21T03:26:35,329 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/C in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:35,329 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/6416548be1af438b86b9ec4ccca94247, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0f10b0f4fc6948d0b86de5327bf530bc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1677cf8b98049b3ae94a9ce443ad8ce] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=35.6 K 2024-11-21T03:26:35,331 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 6416548be1af438b86b9ec4ccca94247, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732159591250 2024-11-21T03:26:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112184a846e958af453896aad5355df6d156_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159593811/Put/seqid=0 2024-11-21T03:26:35,332 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f10b0f4fc6948d0b86de5327bf530bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732159591931 2024-11-21T03:26:35,332 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d1677cf8b98049b3ae94a9ce443ad8ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732159592649 2024-11-21T03:26:35,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742459_1635 (size=12304) 2024-11-21T03:26:35,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,344 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112184a846e958af453896aad5355df6d156_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112184a846e958af453896aad5355df6d156_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:35,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0be73d429d814e89a0926e45d7d86889, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:35,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0be73d429d814e89a0926e45d7d86889 is 175, key is test_row_0/A:col10/1732159593811/Put/seqid=0 2024-11-21T03:26:35,358 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#C#compaction#543 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:35,359 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/9c313b11fc6d4898ab4fbb617a056e16 is 50, key is test_row_0/C:col10/1732159593798/Put/seqid=0 2024-11-21T03:26:35,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742460_1636 (size=31105) 2024-11-21T03:26:35,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742461_1637 (size=12493) 2024-11-21T03:26:35,390 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/9c313b11fc6d4898ab4fbb617a056e16 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9c313b11fc6d4898ab4fbb617a056e16 2024-11-21T03:26:35,406 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/C of 0be8e6cfcf7075b5187319303527b737 into 9c313b11fc6d4898ab4fbb617a056e16(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:35,406 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:35,406 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/C, priority=13, startTime=1732159595237; duration=0sec 2024-11-21T03:26:35,406 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:35,406 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:C 2024-11-21T03:26:35,695 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#A#compaction#540 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:35,696 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/f33b997ad91845009304191e0d39e8e5 is 175, key is test_row_0/A:col10/1732159593798/Put/seqid=0 2024-11-21T03:26:35,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742462_1638 (size=31447) 2024-11-21T03:26:35,708 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/f33b997ad91845009304191e0d39e8e5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f33b997ad91845009304191e0d39e8e5 2024-11-21T03:26:35,715 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/A of 0be8e6cfcf7075b5187319303527b737 into f33b997ad91845009304191e0d39e8e5(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:35,715 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:35,715 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/A, priority=13, startTime=1732159595236; duration=0sec 2024-11-21T03:26:35,715 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:35,715 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:A 2024-11-21T03:26:35,773 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0be73d429d814e89a0926e45d7d86889 2024-11-21T03:26:35,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/5f89e8720c714c5ba13fd92e5fe5510e is 50, key is test_row_0/B:col10/1732159593811/Put/seqid=0 2024-11-21T03:26:35,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742463_1639 (size=12151) 2024-11-21T03:26:35,798 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/5f89e8720c714c5ba13fd92e5fe5510e 2024-11-21T03:26:35,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c018569554da4850aa3135ca9feebd4f is 50, key is test_row_0/C:col10/1732159593811/Put/seqid=0 2024-11-21T03:26:35,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742464_1640 (size=12151) 2024-11-21T03:26:35,824 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c018569554da4850aa3135ca9feebd4f 2024-11-21T03:26:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0be73d429d814e89a0926e45d7d86889 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0be73d429d814e89a0926e45d7d86889 2024-11-21T03:26:35,835 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0be73d429d814e89a0926e45d7d86889, entries=150, sequenceid=173, filesize=30.4 K 2024-11-21T03:26:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/5f89e8720c714c5ba13fd92e5fe5510e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5f89e8720c714c5ba13fd92e5fe5510e 2024-11-21T03:26:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,840 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5f89e8720c714c5ba13fd92e5fe5510e, entries=150, sequenceid=173, filesize=11.9 K 2024-11-21T03:26:35,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c018569554da4850aa3135ca9feebd4f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c018569554da4850aa3135ca9feebd4f 2024-11-21T03:26:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,846 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c018569554da4850aa3135ca9feebd4f, entries=150, sequenceid=173, filesize=11.9 K 2024-11-21T03:26:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,847 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 0be8e6cfcf7075b5187319303527b737 in 557ms, sequenceid=173, compaction requested=false 2024-11-21T03:26:35,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:35,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:35,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-21T03:26:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-21T03:26:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-21T03:26:35,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9770 sec 2024-11-21T03:26:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9830 sec 2024-11-21T03:26:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:36,007 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:26:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e635eda8adc342b0b05c1de553662216_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159596006/Put/seqid=0 2024-11-21T03:26:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T03:26:36,028 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:36,028 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-21T03:26:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T03:26:36,031 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:36,032 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:36,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:36,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742466_1642 (size=24758) 2024-11-21T03:26:36,037 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:36,040 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e635eda8adc342b0b05c1de553662216_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e635eda8adc342b0b05c1de553662216_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:36,042 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/1db42d0f59874f06b463baa569415ec6, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:36,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/1db42d0f59874f06b463baa569415ec6 is 175, key is test_row_0/A:col10/1732159596006/Put/seqid=0 2024-11-21T03:26:36,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742465_1641 (size=74395) 2024-11-21T03:26:36,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159656060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159656060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159656062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159656062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159656066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T03:26:36,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159656178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159656178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159656178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159656178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T03:26:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159656184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T03:26:36,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:36,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T03:26:36,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159656389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159656389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159656389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159656389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159656398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/1db42d0f59874f06b463baa569415ec6 2024-11-21T03:26:36,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/405e33984b9e47deb27df6163d31aa5e is 50, key is test_row_0/B:col10/1732159596006/Put/seqid=0 2024-11-21T03:26:36,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742467_1643 (size=12151) 2024-11-21T03:26:36,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/405e33984b9e47deb27df6163d31aa5e 2024-11-21T03:26:36,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/bf8e990e9e4a41e58fde3eab5d750812 is 50, key is test_row_0/C:col10/1732159596006/Put/seqid=0 2024-11-21T03:26:36,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742468_1644 (size=12151) 2024-11-21T03:26:36,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T03:26:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T03:26:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T03:26:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159656698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159656698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159656698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159656698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159656707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:36,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T03:26:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:36,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/bf8e990e9e4a41e58fde3eab5d750812 2024-11-21T03:26:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/1db42d0f59874f06b463baa569415ec6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1db42d0f59874f06b463baa569415ec6 2024-11-21T03:26:36,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1db42d0f59874f06b463baa569415ec6, entries=400, sequenceid=187, filesize=72.7 K 2024-11-21T03:26:36,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/405e33984b9e47deb27df6163d31aa5e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/405e33984b9e47deb27df6163d31aa5e 2024-11-21T03:26:36,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/405e33984b9e47deb27df6163d31aa5e, entries=150, sequenceid=187, filesize=11.9 K 2024-11-21T03:26:36,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/bf8e990e9e4a41e58fde3eab5d750812 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bf8e990e9e4a41e58fde3eab5d750812 2024-11-21T03:26:36,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bf8e990e9e4a41e58fde3eab5d750812, entries=150, sequenceid=187, filesize=11.9 K 2024-11-21T03:26:36,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0be8e6cfcf7075b5187319303527b737 in 925ms, sequenceid=187, compaction requested=true 2024-11-21T03:26:36,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:36,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:36,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:36,933 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:36,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:36,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:36,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:36,933 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:36,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:36,933 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:36,934 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/A is initiating minor compaction (all files) 2024-11-21T03:26:36,934 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/A in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,934 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f33b997ad91845009304191e0d39e8e5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0be73d429d814e89a0926e45d7d86889, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1db42d0f59874f06b463baa569415ec6] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=133.7 K 2024-11-21T03:26:36,934 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,934 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f33b997ad91845009304191e0d39e8e5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0be73d429d814e89a0926e45d7d86889, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1db42d0f59874f06b463baa569415ec6] 2024-11-21T03:26:36,934 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:36,934 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/B is initiating minor compaction (all files) 2024-11-21T03:26:36,934 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/B in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,934 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/d0891e355ec0432eb46fa134f35d553f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5f89e8720c714c5ba13fd92e5fe5510e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/405e33984b9e47deb27df6163d31aa5e] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=35.9 K 2024-11-21T03:26:36,934 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting d0891e355ec0432eb46fa134f35d553f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732159592649 2024-11-21T03:26:36,935 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f89e8720c714c5ba13fd92e5fe5510e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732159593811 2024-11-21T03:26:36,935 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 405e33984b9e47deb27df6163d31aa5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732159596001 2024-11-21T03:26:36,935 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f33b997ad91845009304191e0d39e8e5, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732159592649 2024-11-21T03:26:36,935 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0be73d429d814e89a0926e45d7d86889, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732159593811 2024-11-21T03:26:36,936 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1db42d0f59874f06b463baa569415ec6, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732159595979 2024-11-21T03:26:36,950 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#B#compaction#549 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:36,951 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/261c10f614fa43d79d5df822988a8a7b is 50, key is test_row_0/B:col10/1732159596006/Put/seqid=0 2024-11-21T03:26:36,953 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:36,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-21T03:26:36,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,967 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:26:36,969 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f7a82d106e5942788ee1795f1975b0af_0be8e6cfcf7075b5187319303527b737 store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:36,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742469_1645 (size=12595) 2024-11-21T03:26:36,976 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f7a82d106e5942788ee1795f1975b0af_0be8e6cfcf7075b5187319303527b737, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:36,976 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f7a82d106e5942788ee1795f1975b0af_0be8e6cfcf7075b5187319303527b737 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:36,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121604f5f4843ab48d39f3128d945b340b1_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159596056/Put/seqid=0 2024-11-21T03:26:36,989 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/261c10f614fa43d79d5df822988a8a7b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/261c10f614fa43d79d5df822988a8a7b 2024-11-21T03:26:36,995 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/B of 0be8e6cfcf7075b5187319303527b737 into 261c10f614fa43d79d5df822988a8a7b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:36,995 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:36,995 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/B, priority=13, startTime=1732159596933; duration=0sec 2024-11-21T03:26:36,995 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:36,995 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:B 2024-11-21T03:26:36,995 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:36,996 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:36,996 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/C is initiating minor compaction (all files) 2024-11-21T03:26:36,996 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/C in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:36,997 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9c313b11fc6d4898ab4fbb617a056e16, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c018569554da4850aa3135ca9feebd4f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bf8e990e9e4a41e58fde3eab5d750812] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=35.9 K 2024-11-21T03:26:36,997 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c313b11fc6d4898ab4fbb617a056e16, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732159592649 2024-11-21T03:26:36,997 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting c018569554da4850aa3135ca9feebd4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732159593811 2024-11-21T03:26:36,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742470_1646 (size=4469) 2024-11-21T03:26:36,998 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting bf8e990e9e4a41e58fde3eab5d750812, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732159596001 2024-11-21T03:26:36,999 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#A#compaction#550 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:36,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742471_1647 (size=12304) 2024-11-21T03:26:36,999 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2fc838d1d07b4edea020f147eea33b74 is 175, key is test_row_0/A:col10/1732159596006/Put/seqid=0 2024-11-21T03:26:37,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:37,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742472_1648 (size=31549) 2024-11-21T03:26:37,004 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121604f5f4843ab48d39f3128d945b340b1_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121604f5f4843ab48d39f3128d945b340b1_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:37,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/5432b23832cc45b1baf9f7219480c54f, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:37,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/5432b23832cc45b1baf9f7219480c54f is 175, key is test_row_0/A:col10/1732159596056/Put/seqid=0 2024-11-21T03:26:37,006 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2fc838d1d07b4edea020f147eea33b74 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2fc838d1d07b4edea020f147eea33b74 2024-11-21T03:26:37,010 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/A of 0be8e6cfcf7075b5187319303527b737 into 2fc838d1d07b4edea020f147eea33b74(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:37,010 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:37,010 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/A, priority=13, startTime=1732159596933; duration=0sec 2024-11-21T03:26:37,010 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:37,010 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:A 2024-11-21T03:26:37,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742473_1649 (size=31105) 2024-11-21T03:26:37,012 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#C#compaction#552 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:37,013 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/1af959ce38f84953971740c7fba8be3a is 50, key is test_row_0/C:col10/1732159596006/Put/seqid=0 2024-11-21T03:26:37,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742474_1650 (size=12595) 2024-11-21T03:26:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T03:26:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:37,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:37,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159657218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159657221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159657222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159657224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159657224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159657328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159657337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159657337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159657337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,413 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/5432b23832cc45b1baf9f7219480c54f 2024-11-21T03:26:37,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/a6d96544e04144fc9e1e22a1bbf6bec4 is 50, key is test_row_0/B:col10/1732159596056/Put/seqid=0 2024-11-21T03:26:37,421 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/1af959ce38f84953971740c7fba8be3a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1af959ce38f84953971740c7fba8be3a 2024-11-21T03:26:37,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742475_1651 (size=12151) 2024-11-21T03:26:37,424 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/C of 0be8e6cfcf7075b5187319303527b737 into 1af959ce38f84953971740c7fba8be3a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:37,424 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:37,424 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/C, priority=13, startTime=1732159596933; duration=0sec 2024-11-21T03:26:37,425 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:37,425 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:C 2024-11-21T03:26:37,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159657538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159657548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159657548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159657548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,822 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/a6d96544e04144fc9e1e22a1bbf6bec4 2024-11-21T03:26:37,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/bd65057e6b93403188b6666e65f50b1e is 50, key is test_row_0/C:col10/1732159596056/Put/seqid=0 2024-11-21T03:26:37,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742476_1652 (size=12151) 2024-11-21T03:26:37,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159657847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159657858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159657858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:37,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159657861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:37,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T03:26:38,232 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/bd65057e6b93403188b6666e65f50b1e 2024-11-21T03:26:38,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/5432b23832cc45b1baf9f7219480c54f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5432b23832cc45b1baf9f7219480c54f 2024-11-21T03:26:38,238 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5432b23832cc45b1baf9f7219480c54f, entries=150, sequenceid=212, filesize=30.4 K 2024-11-21T03:26:38,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/a6d96544e04144fc9e1e22a1bbf6bec4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/a6d96544e04144fc9e1e22a1bbf6bec4 2024-11-21T03:26:38,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159658237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,239 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in 0be8e6cfcf7075b5187319303527b737 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:26:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,241 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/a6d96544e04144fc9e1e22a1bbf6bec4, entries=150, sequenceid=212, filesize=11.9 K 2024-11-21T03:26:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/bd65057e6b93403188b6666e65f50b1e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bd65057e6b93403188b6666e65f50b1e 2024-11-21T03:26:38,244 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bd65057e6b93403188b6666e65f50b1e, entries=150, sequenceid=212, filesize=11.9 K 2024-11-21T03:26:38,244 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0be8e6cfcf7075b5187319303527b737 in 1277ms, sequenceid=212, compaction requested=false 2024-11-21T03:26:38,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:38,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:38,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-21T03:26:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-21T03:26:38,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-21T03:26:38,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2140 sec 2024-11-21T03:26:38,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.2190 sec 2024-11-21T03:26:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:38,359 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:38,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:26:38,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112164bcf112be284787b49183c3c7053319_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159598357/Put/seqid=0 2024-11-21T03:26:38,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742477_1653 (size=12304) 2024-11-21T03:26:38,379 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:38,384 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112164bcf112be284787b49183c3c7053319_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112164bcf112be284787b49183c3c7053319_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:38,385 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/c9408f5f452841dd821e2a5bfd676822, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:38,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/c9408f5f452841dd821e2a5bfd676822 is 175, key is test_row_0/A:col10/1732159598357/Put/seqid=0 2024-11-21T03:26:38,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742478_1654 (size=31105) 2024-11-21T03:26:38,389 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/c9408f5f452841dd821e2a5bfd676822 2024-11-21T03:26:38,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/ad14a84922c1459c9af7788a50bbd780 is 50, key is test_row_0/B:col10/1732159598357/Put/seqid=0 2024-11-21T03:26:38,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742479_1655 (size=12151) 2024-11-21T03:26:38,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159658402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159658402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159658407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159658407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159658518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159658518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159658518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159658518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159658728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159658728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159658728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159658728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:38,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/ad14a84922c1459c9af7788a50bbd780 2024-11-21T03:26:38,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/e88655ad650e435bbc6c798c2e9aa9c1 is 50, key is test_row_0/C:col10/1732159598357/Put/seqid=0 2024-11-21T03:26:38,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742480_1656 (size=12151) 2024-11-21T03:26:39,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159659037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159659037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159659037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159659037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/e88655ad650e435bbc6c798c2e9aa9c1 2024-11-21T03:26:39,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/c9408f5f452841dd821e2a5bfd676822 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/c9408f5f452841dd821e2a5bfd676822 2024-11-21T03:26:39,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/c9408f5f452841dd821e2a5bfd676822, entries=150, sequenceid=227, filesize=30.4 K 2024-11-21T03:26:39,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/ad14a84922c1459c9af7788a50bbd780 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ad14a84922c1459c9af7788a50bbd780 2024-11-21T03:26:39,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ad14a84922c1459c9af7788a50bbd780, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T03:26:39,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/e88655ad650e435bbc6c798c2e9aa9c1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/e88655ad650e435bbc6c798c2e9aa9c1 2024-11-21T03:26:39,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/e88655ad650e435bbc6c798c2e9aa9c1, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T03:26:39,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0be8e6cfcf7075b5187319303527b737 in 951ms, sequenceid=227, compaction requested=true 2024-11-21T03:26:39,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:39,311 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:39,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:39,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:39,311 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:39,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:39,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:39,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:39,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:39,314 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:39,314 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/B is initiating minor compaction (all files) 2024-11-21T03:26:39,314 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/B in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:39,315 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/261c10f614fa43d79d5df822988a8a7b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/a6d96544e04144fc9e1e22a1bbf6bec4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ad14a84922c1459c9af7788a50bbd780] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=36.0 K 2024-11-21T03:26:39,315 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:39,315 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/A is initiating minor compaction (all files) 2024-11-21T03:26:39,315 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/A in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:39,315 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2fc838d1d07b4edea020f147eea33b74, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5432b23832cc45b1baf9f7219480c54f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/c9408f5f452841dd821e2a5bfd676822] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=91.6 K 2024-11-21T03:26:39,315 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:39,315 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2fc838d1d07b4edea020f147eea33b74, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5432b23832cc45b1baf9f7219480c54f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/c9408f5f452841dd821e2a5bfd676822] 2024-11-21T03:26:39,315 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fc838d1d07b4edea020f147eea33b74, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732159596001 2024-11-21T03:26:39,315 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 261c10f614fa43d79d5df822988a8a7b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732159596001 2024-11-21T03:26:39,316 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting a6d96544e04144fc9e1e22a1bbf6bec4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732159596046 2024-11-21T03:26:39,316 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5432b23832cc45b1baf9f7219480c54f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732159596046 2024-11-21T03:26:39,320 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting ad14a84922c1459c9af7788a50bbd780, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732159597221 2024-11-21T03:26:39,320 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9408f5f452841dd821e2a5bfd676822, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732159597221 2024-11-21T03:26:39,337 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#B#compaction#558 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:39,337 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/3db12c4bb56940faa4aac50abcf188ab is 50, key is test_row_0/B:col10/1732159598357/Put/seqid=0 2024-11-21T03:26:39,341 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:39,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742481_1657 (size=12697) 2024-11-21T03:26:39,361 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411210ea74c3476244d23aaeccecc0604a66f_0be8e6cfcf7075b5187319303527b737 store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:39,363 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411210ea74c3476244d23aaeccecc0604a66f_0be8e6cfcf7075b5187319303527b737, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:39,363 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210ea74c3476244d23aaeccecc0604a66f_0be8e6cfcf7075b5187319303527b737 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:39,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742482_1658 (size=4469) 2024-11-21T03:26:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:39,556 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:39,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T03:26:39,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e7a03338d89b46b8bfe01146d7648993_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159598400/Put/seqid=0 2024-11-21T03:26:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159659565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159659565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742483_1659 (size=17284) 2024-11-21T03:26:39,574 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:39,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e7a03338d89b46b8bfe01146d7648993_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e7a03338d89b46b8bfe01146d7648993_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:39,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159659572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159659573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,579 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2bf99c1a89a747bc8108435b37c4596e, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:39,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2bf99c1a89a747bc8108435b37c4596e is 175, key is test_row_0/A:col10/1732159598400/Put/seqid=0 2024-11-21T03:26:39,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742484_1660 (size=48389) 2024-11-21T03:26:39,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159659678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159659678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159659687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159659687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,775 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/3db12c4bb56940faa4aac50abcf188ab as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/3db12c4bb56940faa4aac50abcf188ab 2024-11-21T03:26:39,785 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/B of 0be8e6cfcf7075b5187319303527b737 into 3db12c4bb56940faa4aac50abcf188ab(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:39,785 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:39,785 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/B, priority=13, startTime=1732159599311; duration=0sec 2024-11-21T03:26:39,785 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:39,785 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:B 2024-11-21T03:26:39,785 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:39,797 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:39,797 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/C is initiating minor compaction (all files) 2024-11-21T03:26:39,797 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/C in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:39,797 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1af959ce38f84953971740c7fba8be3a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bd65057e6b93403188b6666e65f50b1e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/e88655ad650e435bbc6c798c2e9aa9c1] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=36.0 K 2024-11-21T03:26:39,798 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1af959ce38f84953971740c7fba8be3a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732159596001 2024-11-21T03:26:39,799 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#A#compaction#559 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:39,799 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/ed87dea1c79742b28fd08ad08da00b33 is 175, key is test_row_0/A:col10/1732159598357/Put/seqid=0 2024-11-21T03:26:39,800 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting bd65057e6b93403188b6666e65f50b1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732159596046 2024-11-21T03:26:39,801 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting e88655ad650e435bbc6c798c2e9aa9c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732159597221 2024-11-21T03:26:39,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742485_1661 (size=31651) 2024-11-21T03:26:39,830 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#C#compaction#561 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:39,830 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/5ab55bf5ea404a27a914209a2a66194a is 50, key is test_row_0/C:col10/1732159598357/Put/seqid=0 2024-11-21T03:26:39,837 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/ed87dea1c79742b28fd08ad08da00b33 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/ed87dea1c79742b28fd08ad08da00b33 2024-11-21T03:26:39,844 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/A of 0be8e6cfcf7075b5187319303527b737 into ed87dea1c79742b28fd08ad08da00b33(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:39,844 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:39,844 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/A, priority=13, startTime=1732159599310; duration=0sec 2024-11-21T03:26:39,844 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:39,845 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:A 2024-11-21T03:26:39,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742486_1662 (size=12697) 2024-11-21T03:26:39,869 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/5ab55bf5ea404a27a914209a2a66194a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/5ab55bf5ea404a27a914209a2a66194a 2024-11-21T03:26:39,884 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/C of 0be8e6cfcf7075b5187319303527b737 into 5ab55bf5ea404a27a914209a2a66194a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:39,884 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:39,884 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/C, priority=13, startTime=1732159599311; duration=0sec 2024-11-21T03:26:39,884 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:39,884 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:C 2024-11-21T03:26:39,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159659888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159659892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159659898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159659898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:39,984 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2bf99c1a89a747bc8108435b37c4596e 2024-11-21T03:26:39,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/4f90847d71c74b04b7325fb5b1e6d034 is 50, key is test_row_0/B:col10/1732159598400/Put/seqid=0 2024-11-21T03:26:39,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742487_1663 (size=12151) 2024-11-21T03:26:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-21T03:26:40,187 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:40,188 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-21T03:26:40,189 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T03:26:40,189 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:40,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:40,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159660197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159660207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159660208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159660208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159660267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,270 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to b69e2a1ee061,45649,1732159441548 for 'test_row_2' in 0be8e6cfcf7075b5187319303527b737 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T03:26:40,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T03:26:40,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T03:26:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:40,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:40,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:40,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:40,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/4f90847d71c74b04b7325fb5b1e6d034 2024-11-21T03:26:40,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/83a627f918ef4667a925d50647b9f8a2 is 50, key is test_row_0/C:col10/1732159598400/Put/seqid=0 2024-11-21T03:26:40,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742488_1664 (size=12151) 2024-11-21T03:26:40,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/83a627f918ef4667a925d50647b9f8a2 2024-11-21T03:26:40,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2bf99c1a89a747bc8108435b37c4596e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2bf99c1a89a747bc8108435b37c4596e 2024-11-21T03:26:40,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2bf99c1a89a747bc8108435b37c4596e, entries=250, sequenceid=253, filesize=47.3 K 2024-11-21T03:26:40,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/4f90847d71c74b04b7325fb5b1e6d034 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4f90847d71c74b04b7325fb5b1e6d034 2024-11-21T03:26:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T03:26:40,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T03:26:40,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:40,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:40,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:40,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:40,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:40,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4f90847d71c74b04b7325fb5b1e6d034, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T03:26:40,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/83a627f918ef4667a925d50647b9f8a2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/83a627f918ef4667a925d50647b9f8a2 2024-11-21T03:26:40,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/83a627f918ef4667a925d50647b9f8a2, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T03:26:40,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0be8e6cfcf7075b5187319303527b737 in 1015ms, sequenceid=253, compaction requested=false 2024-11-21T03:26:40,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:40,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T03:26:40,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:40,669 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T03:26:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210838fa8210e4470597b8c0b7c9c988a6_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159599560/Put/seqid=0 2024-11-21T03:26:40,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742489_1665 (size=12454) 2024-11-21T03:26:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:40,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159660749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159660750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159660750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159660751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T03:26:40,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159660857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159660857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159660857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159660857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159661067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159661067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159661068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159661068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:41,108 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210838fa8210e4470597b8c0b7c9c988a6_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210838fa8210e4470597b8c0b7c9c988a6_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:41,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/95fcb719bbb04a1db125e4d5d81539a2, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:41,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/95fcb719bbb04a1db125e4d5d81539a2 is 175, key is test_row_0/A:col10/1732159599560/Put/seqid=0 2024-11-21T03:26:41,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742490_1666 (size=31255) 2024-11-21T03:26:41,116 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/95fcb719bbb04a1db125e4d5d81539a2 2024-11-21T03:26:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/5de13b3d86be43cab074d156f5ecfc4c is 50, key is test_row_0/B:col10/1732159599560/Put/seqid=0 2024-11-21T03:26:41,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742491_1667 (size=12301) 2024-11-21T03:26:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T03:26:41,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159661378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159661378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159661378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159661378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,524 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/5de13b3d86be43cab074d156f5ecfc4c 2024-11-21T03:26:41,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/7e0c664b94cb4fbf963867f9ae4e2bc6 is 50, key is test_row_0/C:col10/1732159599560/Put/seqid=0 2024-11-21T03:26:41,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742492_1668 (size=12301) 2024-11-21T03:26:41,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159661888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159661894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159661894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159661897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:41,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:41,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:41,942 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/7e0c664b94cb4fbf963867f9ae4e2bc6 2024-11-21T03:26:41,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/95fcb719bbb04a1db125e4d5d81539a2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/95fcb719bbb04a1db125e4d5d81539a2 2024-11-21T03:26:41,962 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/95fcb719bbb04a1db125e4d5d81539a2, entries=150, sequenceid=266, filesize=30.5 K 2024-11-21T03:26:41,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/5de13b3d86be43cab074d156f5ecfc4c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5de13b3d86be43cab074d156f5ecfc4c 2024-11-21T03:26:41,965 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5de13b3d86be43cab074d156f5ecfc4c, entries=150, sequenceid=266, filesize=12.0 K 2024-11-21T03:26:41,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/7e0c664b94cb4fbf963867f9ae4e2bc6 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/7e0c664b94cb4fbf963867f9ae4e2bc6 2024-11-21T03:26:41,969 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/7e0c664b94cb4fbf963867f9ae4e2bc6, entries=150, sequenceid=266, filesize=12.0 K 2024-11-21T03:26:41,970 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 0be8e6cfcf7075b5187319303527b737 in 1300ms, sequenceid=266, compaction requested=true 2024-11-21T03:26:41,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:41,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:41,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-21T03:26:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-21T03:26:41,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-21T03:26:41,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7900 sec 2024-11-21T03:26:41,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8060 sec 2024-11-21T03:26:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T03:26:42,338 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:42,339 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-21T03:26:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T03:26:42,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:42,349 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:42,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T03:26:42,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T03:26:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:42,509 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T03:26:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112143f223dd48a847f8a03814b9fdaedd23_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159600749/Put/seqid=0 2024-11-21T03:26:42,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742493_1669 (size=12454) 2024-11-21T03:26:42,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,526 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112143f223dd48a847f8a03814b9fdaedd23_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112143f223dd48a847f8a03814b9fdaedd23_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2627f34811a94b6f884378361d37427d, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2627f34811a94b6f884378361d37427d is 175, key is test_row_0/A:col10/1732159600749/Put/seqid=0 2024-11-21T03:26:42,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742494_1670 (size=31255) 2024-11-21T03:26:42,535 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2627f34811a94b6f884378361d37427d 2024-11-21T03:26:42,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/8f9e8d412aaa4cdfb820bab493704c7c is 50, key is test_row_0/B:col10/1732159600749/Put/seqid=0 2024-11-21T03:26:42,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742495_1671 (size=12301) 2024-11-21T03:26:42,545 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/8f9e8d412aaa4cdfb820bab493704c7c 2024-11-21T03:26:42,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/9111f4afa2a244a1aa7dfcfb0b010f46 is 50, key is test_row_0/C:col10/1732159600749/Put/seqid=0 2024-11-21T03:26:42,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742496_1672 (size=12301) 2024-11-21T03:26:42,558 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/9111f4afa2a244a1aa7dfcfb0b010f46 2024-11-21T03:26:42,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/2627f34811a94b6f884378361d37427d as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2627f34811a94b6f884378361d37427d 2024-11-21T03:26:42,569 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2627f34811a94b6f884378361d37427d, entries=150, sequenceid=292, filesize=30.5 K 2024-11-21T03:26:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/8f9e8d412aaa4cdfb820bab493704c7c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f9e8d412aaa4cdfb820bab493704c7c 2024-11-21T03:26:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,573 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f9e8d412aaa4cdfb820bab493704c7c, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T03:26:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/9111f4afa2a244a1aa7dfcfb0b010f46 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9111f4afa2a244a1aa7dfcfb0b010f46 2024-11-21T03:26:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,577 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9111f4afa2a244a1aa7dfcfb0b010f46, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T03:26:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,578 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 0be8e6cfcf7075b5187319303527b737 in 69ms, sequenceid=292, compaction requested=true 2024-11-21T03:26:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:42,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-21T03:26:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-21T03:26:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-21T03:26:42,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 230 msec 2024-11-21T03:26:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 244 msec 2024-11-21T03:26:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T03:26:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,661 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,664 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-21T03:26:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T03:26:42,666 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:42,667 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T03:26:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T03:26:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:42,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:42,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-21T03:26:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-21T03:26:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-21T03:26:42,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 166 msec 2024-11-21T03:26:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 173 msec 2024-11-21T03:26:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T03:26:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,996 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:43,004 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:26:43,005 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:43,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:26:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-21T03:26:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,014 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:26:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T03:26:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,016 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:26:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:26:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112137df705069e342bcb67cd0275ad4aafb_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159602948/Put/seqid=0 2024-11-21T03:26:43,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742497_1673 (size=20074) 2024-11-21T03:26:43,090 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:43,121 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112137df705069e342bcb67cd0275ad4aafb_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112137df705069e342bcb67cd0275ad4aafb_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:43,122 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0648079ad3704413858eeb12d35923d5, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:43,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0648079ad3704413858eeb12d35923d5 is 175, key is test_row_0/A:col10/1732159602948/Put/seqid=0 2024-11-21T03:26:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T03:26:43,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159663118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159663139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159663141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159663144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742498_1674 (size=57333) 2024-11-21T03:26:43,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159663252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159663260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159663260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159663271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:43,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:43,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T03:26:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159663468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159663479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159663479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159663479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:43,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:43,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,567 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0648079ad3704413858eeb12d35923d5 2024-11-21T03:26:43,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/03d12433a6c64f91b55baf821a421190 is 50, key is test_row_0/B:col10/1732159602948/Put/seqid=0 2024-11-21T03:26:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T03:26:43,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742499_1675 (size=12301) 2024-11-21T03:26:43,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/03d12433a6c64f91b55baf821a421190 2024-11-21T03:26:43,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:43,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:43,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/f42e76e3a7264152a91a8d519c4c7de5 is 50, key is test_row_0/C:col10/1732159602948/Put/seqid=0 2024-11-21T03:26:43,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742500_1676 (size=12301) 2024-11-21T03:26:43,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159663777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159663787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159663798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159663798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:43,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:43,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:43,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/f42e76e3a7264152a91a8d519c4c7de5 2024-11-21T03:26:44,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/0648079ad3704413858eeb12d35923d5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0648079ad3704413858eeb12d35923d5 2024-11-21T03:26:44,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0648079ad3704413858eeb12d35923d5, entries=300, sequenceid=304, filesize=56.0 K 2024-11-21T03:26:44,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/03d12433a6c64f91b55baf821a421190 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/03d12433a6c64f91b55baf821a421190 2024-11-21T03:26:44,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/03d12433a6c64f91b55baf821a421190, entries=150, sequenceid=304, filesize=12.0 K 2024-11-21T03:26:44,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/f42e76e3a7264152a91a8d519c4c7de5 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/f42e76e3a7264152a91a8d519c4c7de5 2024-11-21T03:26:44,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/f42e76e3a7264152a91a8d519c4c7de5, entries=150, sequenceid=304, filesize=12.0 K 2024-11-21T03:26:44,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=161.02 KB/164880 for 0be8e6cfcf7075b5187319303527b737 in 1149ms, sequenceid=304, compaction requested=true 2024-11-21T03:26:44,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:44,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:44,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:44,154 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T03:26:44,154 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T03:26:44,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:44,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:44,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:44,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T03:26:44,176 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 199883 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T03:26:44,176 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61751 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T03:26:44,176 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/B is initiating minor compaction (all files) 2024-11-21T03:26:44,176 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/A is initiating minor compaction (all files) 2024-11-21T03:26:44,176 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/A in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,176 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/ed87dea1c79742b28fd08ad08da00b33, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2bf99c1a89a747bc8108435b37c4596e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/95fcb719bbb04a1db125e4d5d81539a2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2627f34811a94b6f884378361d37427d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0648079ad3704413858eeb12d35923d5] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=195.2 K 2024-11-21T03:26:44,177 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,177 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/ed87dea1c79742b28fd08ad08da00b33, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2bf99c1a89a747bc8108435b37c4596e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/95fcb719bbb04a1db125e4d5d81539a2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2627f34811a94b6f884378361d37427d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0648079ad3704413858eeb12d35923d5] 2024-11-21T03:26:44,177 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/B in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,177 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/3db12c4bb56940faa4aac50abcf188ab, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4f90847d71c74b04b7325fb5b1e6d034, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5de13b3d86be43cab074d156f5ecfc4c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f9e8d412aaa4cdfb820bab493704c7c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/03d12433a6c64f91b55baf821a421190] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=60.3 K 2024-11-21T03:26:44,181 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db12c4bb56940faa4aac50abcf188ab, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732159597221 2024-11-21T03:26:44,182 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed87dea1c79742b28fd08ad08da00b33, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732159597221 2024-11-21T03:26:44,183 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f90847d71c74b04b7325fb5b1e6d034, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732159598400 2024-11-21T03:26:44,183 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bf99c1a89a747bc8108435b37c4596e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732159598400 2024-11-21T03:26:44,183 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de13b3d86be43cab074d156f5ecfc4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732159599560 2024-11-21T03:26:44,183 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95fcb719bbb04a1db125e4d5d81539a2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732159599560 2024-11-21T03:26:44,184 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2627f34811a94b6f884378361d37427d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732159600743 2024-11-21T03:26:44,184 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f9e8d412aaa4cdfb820bab493704c7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732159600743 2024-11-21T03:26:44,191 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d12433a6c64f91b55baf821a421190, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159602948 2024-11-21T03:26:44,191 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0648079ad3704413858eeb12d35923d5, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159602948 2024-11-21T03:26:44,207 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:44,208 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#B#compaction#573 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:44,208 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/2023608fbae941ea8fe51efb62c44fc8 is 50, key is test_row_0/B:col10/1732159602948/Put/seqid=0 2024-11-21T03:26:44,234 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411214701f3da75724931a213b310a1e651cf_0be8e6cfcf7075b5187319303527b737 store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:44,238 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411214701f3da75724931a213b310a1e651cf_0be8e6cfcf7075b5187319303527b737, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:44,238 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214701f3da75724931a213b310a1e651cf_0be8e6cfcf7075b5187319303527b737 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:44,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742501_1677 (size=13017) 2024-11-21T03:26:44,253 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/2023608fbae941ea8fe51efb62c44fc8 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2023608fbae941ea8fe51efb62c44fc8 2024-11-21T03:26:44,258 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/B of 0be8e6cfcf7075b5187319303527b737 into 2023608fbae941ea8fe51efb62c44fc8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:44,258 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:44,258 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/B, priority=11, startTime=1732159604154; duration=0sec 2024-11-21T03:26:44,258 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:44,258 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:B 2024-11-21T03:26:44,258 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T03:26:44,259 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61751 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T03:26:44,259 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/C is initiating minor compaction (all files) 2024-11-21T03:26:44,259 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/C in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,259 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/5ab55bf5ea404a27a914209a2a66194a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/83a627f918ef4667a925d50647b9f8a2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/7e0c664b94cb4fbf963867f9ae4e2bc6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9111f4afa2a244a1aa7dfcfb0b010f46, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/f42e76e3a7264152a91a8d519c4c7de5] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=60.3 K 2024-11-21T03:26:44,260 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ab55bf5ea404a27a914209a2a66194a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732159597221 2024-11-21T03:26:44,260 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 83a627f918ef4667a925d50647b9f8a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732159598400 2024-11-21T03:26:44,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742502_1678 (size=4469) 2024-11-21T03:26:44,261 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e0c664b94cb4fbf963867f9ae4e2bc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732159599560 2024-11-21T03:26:44,262 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9111f4afa2a244a1aa7dfcfb0b010f46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732159600743 2024-11-21T03:26:44,262 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting f42e76e3a7264152a91a8d519c4c7de5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159602948 2024-11-21T03:26:44,271 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#C#compaction#575 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:44,271 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/238eacb033b44d188666b098558991ca is 50, key is test_row_0/C:col10/1732159602948/Put/seqid=0 2024-11-21T03:26:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:44,292 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:44,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-21T03:26:44,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:44,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:44,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742503_1679 (size=13017) 2024-11-21T03:26:44,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213b6b258fc99941bf9daca1dd4c8638c9_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159603130/Put/seqid=0 2024-11-21T03:26:44,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159664309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159664309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159664309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159664309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159664320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,331 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/238eacb033b44d188666b098558991ca as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/238eacb033b44d188666b098558991ca 2024-11-21T03:26:44,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742504_1680 (size=14994) 2024-11-21T03:26:44,354 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:44,360 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/C of 0be8e6cfcf7075b5187319303527b737 into 238eacb033b44d188666b098558991ca(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:44,360 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:44,360 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/C, priority=11, startTime=1732159604156; duration=0sec 2024-11-21T03:26:44,360 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:44,360 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:C 2024-11-21T03:26:44,370 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213b6b258fc99941bf9daca1dd4c8638c9_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213b6b258fc99941bf9daca1dd4c8638c9_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:44,377 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/b5fcccd1a1564848a1f43aba88fc891e, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:44,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/b5fcccd1a1564848a1f43aba88fc891e is 175, key is test_row_0/A:col10/1732159603130/Put/seqid=0 2024-11-21T03:26:44,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742505_1681 (size=39949) 2024-11-21T03:26:44,416 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/b5fcccd1a1564848a1f43aba88fc891e 2024-11-21T03:26:44,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159664428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159664428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159664438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/ca23fc01f0fb492595a6af1cadc4f312 is 50, key is test_row_0/B:col10/1732159603130/Put/seqid=0 2024-11-21T03:26:44,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742506_1682 (size=12301) 2024-11-21T03:26:44,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:44,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:44,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:44,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159664637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159664637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159664647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,663 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#A#compaction#574 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:44,664 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/fdc53eb1459e45d7b54067d89b9fd6a3 is 175, key is test_row_0/A:col10/1732159602948/Put/seqid=0 2024-11-21T03:26:44,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742507_1683 (size=31971) 2024-11-21T03:26:44,681 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/fdc53eb1459e45d7b54067d89b9fd6a3 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/fdc53eb1459e45d7b54067d89b9fd6a3 2024-11-21T03:26:44,685 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/A of 0be8e6cfcf7075b5187319303527b737 into fdc53eb1459e45d7b54067d89b9fd6a3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:44,686 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:44,686 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/A, priority=11, startTime=1732159604154; duration=0sec 2024-11-21T03:26:44,686 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:44,686 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:A 2024-11-21T03:26:44,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:44,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/ca23fc01f0fb492595a6af1cadc4f312 2024-11-21T03:26:44,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/8b191843fef4458b8fa9bbef3f662851 is 50, key is test_row_0/C:col10/1732159603130/Put/seqid=0 2024-11-21T03:26:44,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742508_1684 (size=12301) 2024-11-21T03:26:44,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:44,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:44,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:44,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:44,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159664948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159664948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:44,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:44,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159664957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:45,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:45,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:45,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T03:26:45,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/8b191843fef4458b8fa9bbef3f662851 2024-11-21T03:26:45,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/b5fcccd1a1564848a1f43aba88fc891e as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b5fcccd1a1564848a1f43aba88fc891e 2024-11-21T03:26:45,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b5fcccd1a1564848a1f43aba88fc891e, entries=200, sequenceid=333, filesize=39.0 K 2024-11-21T03:26:45,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/ca23fc01f0fb492595a6af1cadc4f312 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ca23fc01f0fb492595a6af1cadc4f312 2024-11-21T03:26:45,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ca23fc01f0fb492595a6af1cadc4f312, entries=150, sequenceid=333, filesize=12.0 K 2024-11-21T03:26:45,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/8b191843fef4458b8fa9bbef3f662851 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/8b191843fef4458b8fa9bbef3f662851 2024-11-21T03:26:45,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/8b191843fef4458b8fa9bbef3f662851, entries=150, sequenceid=333, filesize=12.0 K 2024-11-21T03:26:45,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 0be8e6cfcf7075b5187319303527b737 in 1018ms, sequenceid=333, compaction requested=false 2024-11-21T03:26:45,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:45,335 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:45,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:26:45,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ce56752dcd9042ac91671770eb9075b6_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159605331/Put/seqid=0 2024-11-21T03:26:45,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742509_1685 (size=14994) 2024-11-21T03:26:45,345 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:45,348 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ce56752dcd9042ac91671770eb9075b6_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ce56752dcd9042ac91671770eb9075b6_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:45,349 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/157b528c740c40c98cb848983ff68aba, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:45,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/157b528c740c40c98cb848983ff68aba is 175, key is test_row_0/A:col10/1732159605331/Put/seqid=0 2024-11-21T03:26:45,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742510_1686 (size=39949) 2024-11-21T03:26:45,352 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/157b528c740c40c98cb848983ff68aba 2024-11-21T03:26:45,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/2da526dc923c4cd994e576271c912a31 is 50, key is test_row_0/B:col10/1732159605331/Put/seqid=0 2024-11-21T03:26:45,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742511_1687 (size=12301) 2024-11-21T03:26:45,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:45,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:45,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159665434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:45,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:45,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159665437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:45,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159665457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:45,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159665458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:45,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159665468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,472 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:45,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159665548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159665548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:45,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:45,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:45,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:45,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:45,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159665758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:45,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/2da526dc923c4cd994e576271c912a31 2024-11-21T03:26:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159665758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:45,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/0689c102a771457e8df984ca810b5be1 is 50, key is test_row_0/C:col10/1732159605331/Put/seqid=0 2024-11-21T03:26:45,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742512_1688 (size=12301) 2024-11-21T03:26:45,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:45,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:45,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:45,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:46,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:46,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:46,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159666068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159666068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/0689c102a771457e8df984ca810b5be1 2024-11-21T03:26:46,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:46,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:46,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:46,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/157b528c740c40c98cb848983ff68aba as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/157b528c740c40c98cb848983ff68aba 2024-11-21T03:26:46,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:46,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:26:46,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/157b528c740c40c98cb848983ff68aba, entries=200, sequenceid=346, filesize=39.0 K 2024-11-21T03:26:46,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/2da526dc923c4cd994e576271c912a31 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2da526dc923c4cd994e576271c912a31 2024-11-21T03:26:46,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2da526dc923c4cd994e576271c912a31, entries=150, sequenceid=346, filesize=12.0 K 2024-11-21T03:26:46,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/0689c102a771457e8df984ca810b5be1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0689c102a771457e8df984ca810b5be1 2024-11-21T03:26:46,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0689c102a771457e8df984ca810b5be1, entries=150, sequenceid=346, filesize=12.0 K 2024-11-21T03:26:46,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0be8e6cfcf7075b5187319303527b737 in 865ms, sequenceid=346, compaction requested=true 2024-11-21T03:26:46,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:46,201 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:46,201 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:46,201 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:46,201 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:46,201 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/A is initiating minor compaction (all files) 2024-11-21T03:26:46,201 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/B is initiating minor compaction (all files) 2024-11-21T03:26:46,202 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/B in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:46,202 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/A in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:46,202 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2023608fbae941ea8fe51efb62c44fc8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ca23fc01f0fb492595a6af1cadc4f312, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2da526dc923c4cd994e576271c912a31] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=36.7 K 2024-11-21T03:26:46,202 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/fdc53eb1459e45d7b54067d89b9fd6a3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b5fcccd1a1564848a1f43aba88fc891e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/157b528c740c40c98cb848983ff68aba] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=109.2 K 2024-11-21T03:26:46,202 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:46,202 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/fdc53eb1459e45d7b54067d89b9fd6a3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b5fcccd1a1564848a1f43aba88fc891e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/157b528c740c40c98cb848983ff68aba] 2024-11-21T03:26:46,203 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2023608fbae941ea8fe51efb62c44fc8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159602948 2024-11-21T03:26:46,203 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc53eb1459e45d7b54067d89b9fd6a3, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159602948 2024-11-21T03:26:46,203 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting ca23fc01f0fb492595a6af1cadc4f312, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732159603128 2024-11-21T03:26:46,204 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5fcccd1a1564848a1f43aba88fc891e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732159603075 2024-11-21T03:26:46,204 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 157b528c740c40c98cb848983ff68aba, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732159604302 2024-11-21T03:26:46,204 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da526dc923c4cd994e576271c912a31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732159605328 2024-11-21T03:26:46,209 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:46,210 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#B#compaction#582 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:46,210 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/99eaf0360cea4c4aaf5121dca5e83b8c is 50, key is test_row_0/B:col10/1732159605331/Put/seqid=0 2024-11-21T03:26:46,217 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112175bb1488eeb2495fa237d1922ea0b196_0be8e6cfcf7075b5187319303527b737 store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:46,219 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112175bb1488eeb2495fa237d1922ea0b196_0be8e6cfcf7075b5187319303527b737, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:46,219 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112175bb1488eeb2495fa237d1922ea0b196_0be8e6cfcf7075b5187319303527b737 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:46,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742513_1689 (size=13119) 2024-11-21T03:26:46,243 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/99eaf0360cea4c4aaf5121dca5e83b8c as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/99eaf0360cea4c4aaf5121dca5e83b8c 2024-11-21T03:26:46,247 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/B of 0be8e6cfcf7075b5187319303527b737 into 99eaf0360cea4c4aaf5121dca5e83b8c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:46,247 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:46,247 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/B, priority=13, startTime=1732159606201; duration=0sec 2024-11-21T03:26:46,248 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:46,248 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:B 2024-11-21T03:26:46,248 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:46,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742514_1690 (size=4469) 2024-11-21T03:26:46,252 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:46,252 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/C is initiating minor compaction (all files) 2024-11-21T03:26:46,252 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/C in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:46,252 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/238eacb033b44d188666b098558991ca, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/8b191843fef4458b8fa9bbef3f662851, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0689c102a771457e8df984ca810b5be1] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=36.7 K 2024-11-21T03:26:46,256 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#A#compaction#583 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:46,257 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/1370cc916a1a41daa1a84f0e7d01cb58 is 175, key is test_row_0/A:col10/1732159605331/Put/seqid=0 2024-11-21T03:26:46,259 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 238eacb033b44d188666b098558991ca, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732159602948 2024-11-21T03:26:46,263 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b191843fef4458b8fa9bbef3f662851, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732159603128 2024-11-21T03:26:46,269 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 0689c102a771457e8df984ca810b5be1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732159605328 2024-11-21T03:26:46,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742515_1691 (size=32073) 2024-11-21T03:26:46,310 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#C#compaction#584 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:46,311 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/9e8f167e3f404cafb682756e21f62e41 is 50, key is test_row_0/C:col10/1732159605331/Put/seqid=0 2024-11-21T03:26:46,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45649 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T03:26:46,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:46,341 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:26:46,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742516_1692 (size=13119) 2024-11-21T03:26:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112134b128331a574fa9992684975c4fd8eb_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159605427/Put/seqid=0 2024-11-21T03:26:46,384 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/9e8f167e3f404cafb682756e21f62e41 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9e8f167e3f404cafb682756e21f62e41 2024-11-21T03:26:46,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742517_1693 (size=12454) 2024-11-21T03:26:46,393 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/C of 0be8e6cfcf7075b5187319303527b737 into 9e8f167e3f404cafb682756e21f62e41(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:46,393 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:46,393 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/C, priority=13, startTime=1732159606201; duration=0sec 2024-11-21T03:26:46,393 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:46,393 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:C 2024-11-21T03:26:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:46,417 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112134b128331a574fa9992684975c4fd8eb_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112134b128331a574fa9992684975c4fd8eb_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:46,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/b51f5839d1a340759d1a8b7ffe0b2212, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:46,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/b51f5839d1a340759d1a8b7ffe0b2212 is 175, key is test_row_0/A:col10/1732159605427/Put/seqid=0 2024-11-21T03:26:46,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742518_1694 (size=31255) 2024-11-21T03:26:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:46,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. as already flushing 2024-11-21T03:26:46,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159666484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159666489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159666489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159666580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159666588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159666599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159666599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159666599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,692 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/1370cc916a1a41daa1a84f0e7d01cb58 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1370cc916a1a41daa1a84f0e7d01cb58 2024-11-21T03:26:46,697 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/A of 0be8e6cfcf7075b5187319303527b737 into 1370cc916a1a41daa1a84f0e7d01cb58(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:46,697 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:46,697 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/A, priority=13, startTime=1732159606201; duration=0sec 2024-11-21T03:26:46,697 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:46,697 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:A 2024-11-21T03:26:46,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159666808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159666808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159666808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:46,849 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/b51f5839d1a340759d1a8b7ffe0b2212 2024-11-21T03:26:46,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/8f58ea91bb3349b8871de78b7ab72391 is 50, key is test_row_0/B:col10/1732159605427/Put/seqid=0 2024-11-21T03:26:46,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742519_1695 (size=12301) 2024-11-21T03:26:47,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159667118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159667118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159667118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T03:26:47,329 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/8f58ea91bb3349b8871de78b7ab72391 2024-11-21T03:26:47,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/be62cf1ee4b14fa49673dee31dfa3d2f is 50, key is test_row_0/C:col10/1732159605427/Put/seqid=0 2024-11-21T03:26:47,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742520_1696 (size=12301) 2024-11-21T03:26:47,346 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/be62cf1ee4b14fa49673dee31dfa3d2f 2024-11-21T03:26:47,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/b51f5839d1a340759d1a8b7ffe0b2212 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b51f5839d1a340759d1a8b7ffe0b2212 2024-11-21T03:26:47,354 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b51f5839d1a340759d1a8b7ffe0b2212, entries=150, sequenceid=372, filesize=30.5 K 2024-11-21T03:26:47,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/8f58ea91bb3349b8871de78b7ab72391 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f58ea91bb3349b8871de78b7ab72391 2024-11-21T03:26:47,359 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f58ea91bb3349b8871de78b7ab72391, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T03:26:47,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/be62cf1ee4b14fa49673dee31dfa3d2f as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/be62cf1ee4b14fa49673dee31dfa3d2f 2024-11-21T03:26:47,365 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/be62cf1ee4b14fa49673dee31dfa3d2f, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T03:26:47,366 INFO [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 0be8e6cfcf7075b5187319303527b737 in 1025ms, sequenceid=372, compaction requested=false 2024-11-21T03:26:47,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:47,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:47,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b69e2a1ee061:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-21T03:26:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-21T03:26:47,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-21T03:26:47,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3720 sec 2024-11-21T03:26:47,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 4.4160 sec 2024-11-21T03:26:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:47,591 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:47,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:26:47,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121243dae681d8b42c3a3615f38d7cd5eba_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159606487/Put/seqid=0 2024-11-21T03:26:47,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742521_1697 (size=14994) 2024-11-21T03:26:47,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159667662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159667661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159667666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159667665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159667673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159667777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159667788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159667788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159667788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159667788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159667988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:47,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159667998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159668000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159668000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159668002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,017 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:48,045 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121243dae681d8b42c3a3615f38d7cd5eba_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121243dae681d8b42c3a3615f38d7cd5eba_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:48,046 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/f13095b4da5244f68f475f1660dc2da1, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:48,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/f13095b4da5244f68f475f1660dc2da1 is 175, key is test_row_0/A:col10/1732159606487/Put/seqid=0 2024-11-21T03:26:48,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742522_1698 (size=39949) 2024-11-21T03:26:48,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159668298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159668308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159668318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159668318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159668318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,488 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/f13095b4da5244f68f475f1660dc2da1 2024-11-21T03:26:48,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/63731c76d0854b278ed5e3cb791030c4 is 50, key is test_row_0/B:col10/1732159606487/Put/seqid=0 2024-11-21T03:26:48,569 INFO [Thread-2630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2630. 2024-11-21T03:26:48,569 DEBUG [Thread-2630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:48,570 DEBUG [Thread-2630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:48,570 DEBUG [Thread-2630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:48,571 INFO [Thread-2634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2634. 2024-11-21T03:26:48,571 INFO [Thread-2632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2632. 2024-11-21T03:26:48,571 DEBUG [Thread-2634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:48,571 DEBUG [Thread-2632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:48,571 DEBUG [Thread-2634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:48,571 DEBUG [Thread-2632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:48,571 DEBUG [Thread-2634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:48,571 DEBUG [Thread-2632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:48,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:48,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:48,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:48,575 INFO [Thread-2628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2628. 2024-11-21T03:26:48,575 DEBUG [Thread-2628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:48,575 DEBUG [Thread-2628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:48,575 DEBUG [Thread-2628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:48,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:48,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742523_1699 (size=12301) 2024-11-21T03:26:48,580 INFO [Thread-2636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2636. 2024-11-21T03:26:48,580 DEBUG [Thread-2636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:48,581 DEBUG [Thread-2636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:48,581 DEBUG [Thread-2636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:48,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:48,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40588 deadline: 1732159668808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40616 deadline: 1732159668817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40602 deadline: 1732159668827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T03:26:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40676 deadline: 1732159668838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40646 deadline: 1732159668838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 , the old value is region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:26:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., hostname=b69e2a1ee061,45649,1732159441548, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:26:48,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/63731c76d0854b278ed5e3cb791030c4 2024-11-21T03:26:48,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/1b89772677c24d05aa1272cff1adece0 is 50, key is test_row_0/C:col10/1732159606487/Put/seqid=0 2024-11-21T03:26:49,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742524_1700 (size=12301) 2024-11-21T03:26:49,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/1b89772677c24d05aa1272cff1adece0 2024-11-21T03:26:49,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/f13095b4da5244f68f475f1660dc2da1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f13095b4da5244f68f475f1660dc2da1 2024-11-21T03:26:49,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f13095b4da5244f68f475f1660dc2da1, entries=200, sequenceid=387, filesize=39.0 K 2024-11-21T03:26:49,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/63731c76d0854b278ed5e3cb791030c4 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/63731c76d0854b278ed5e3cb791030c4 2024-11-21T03:26:49,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/63731c76d0854b278ed5e3cb791030c4, entries=150, sequenceid=387, filesize=12.0 K 2024-11-21T03:26:49,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/1b89772677c24d05aa1272cff1adece0 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1b89772677c24d05aa1272cff1adece0 2024-11-21T03:26:49,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1b89772677c24d05aa1272cff1adece0, entries=150, sequenceid=387, filesize=12.0 K 2024-11-21T03:26:49,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0be8e6cfcf7075b5187319303527b737 in 1819ms, sequenceid=387, compaction requested=true 2024-11-21T03:26:49,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:49,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:26:49,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:49,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:26:49,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:49,411 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:49,411 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:49,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be8e6cfcf7075b5187319303527b737:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:26:49,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:49,411 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:49,411 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:49,411 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/B is initiating minor compaction (all files) 2024-11-21T03:26:49,411 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/A is initiating minor compaction (all files) 2024-11-21T03:26:49,412 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/B in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:49,412 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/A in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:49,412 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/99eaf0360cea4c4aaf5121dca5e83b8c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f58ea91bb3349b8871de78b7ab72391, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/63731c76d0854b278ed5e3cb791030c4] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=36.8 K 2024-11-21T03:26:49,412 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1370cc916a1a41daa1a84f0e7d01cb58, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b51f5839d1a340759d1a8b7ffe0b2212, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f13095b4da5244f68f475f1660dc2da1] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=100.9 K 2024-11-21T03:26:49,412 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:49,412 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. files: [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1370cc916a1a41daa1a84f0e7d01cb58, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b51f5839d1a340759d1a8b7ffe0b2212, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f13095b4da5244f68f475f1660dc2da1] 2024-11-21T03:26:49,412 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 99eaf0360cea4c4aaf5121dca5e83b8c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732159605328 2024-11-21T03:26:49,412 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1370cc916a1a41daa1a84f0e7d01cb58, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732159605328 2024-11-21T03:26:49,412 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f58ea91bb3349b8871de78b7ab72391, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732159605421 2024-11-21T03:26:49,412 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51f5839d1a340759d1a8b7ffe0b2212, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732159605421 2024-11-21T03:26:49,412 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 63731c76d0854b278ed5e3cb791030c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732159606482 2024-11-21T03:26:49,412 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] compactions.Compactor(225): Compacting f13095b4da5244f68f475f1660dc2da1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732159606482 2024-11-21T03:26:49,416 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:49,418 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#B#compaction#592 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:49,418 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/bc13ccb8954b44cba04892c0234bcca9 is 50, key is test_row_0/B:col10/1732159606487/Put/seqid=0 2024-11-21T03:26:49,419 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f03227afa0294a08a40e418748f8faa1_0be8e6cfcf7075b5187319303527b737 store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:49,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742525_1701 (size=13221) 2024-11-21T03:26:49,423 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f03227afa0294a08a40e418748f8faa1_0be8e6cfcf7075b5187319303527b737, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:49,423 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f03227afa0294a08a40e418748f8faa1_0be8e6cfcf7075b5187319303527b737 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:49,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742526_1702 (size=4469) 2024-11-21T03:26:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45649 {}] regionserver.HRegion(8855): Flush requested on 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:49,819 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:26:49,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:26:49,819 INFO [Thread-2625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2625. 2024-11-21T03:26:49,819 DEBUG [Thread-2625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:49,819 DEBUG [Thread-2625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:49,819 DEBUG [Thread-2625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:49,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:49,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cfd7e0912ed94de09f21c4cea9ea4b53_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159607660/Put/seqid=0 2024-11-21T03:26:49,825 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/bc13ccb8954b44cba04892c0234bcca9 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/bc13ccb8954b44cba04892c0234bcca9 2024-11-21T03:26:49,828 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/B of 0be8e6cfcf7075b5187319303527b737 into bc13ccb8954b44cba04892c0234bcca9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:49,828 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:49,828 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/B, priority=13, startTime=1732159609411; duration=0sec 2024-11-21T03:26:49,828 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:26:49,828 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:B 2024-11-21T03:26:49,828 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:26:49,829 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:26:49,829 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1541): 0be8e6cfcf7075b5187319303527b737/C is initiating minor compaction (all files) 2024-11-21T03:26:49,829 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be8e6cfcf7075b5187319303527b737/C in TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:49,829 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9e8f167e3f404cafb682756e21f62e41, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/be62cf1ee4b14fa49673dee31dfa3d2f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1b89772677c24d05aa1272cff1adece0] into tmpdir=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp, totalSize=36.8 K 2024-11-21T03:26:49,829 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e8f167e3f404cafb682756e21f62e41, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732159605328 2024-11-21T03:26:49,830 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting be62cf1ee4b14fa49673dee31dfa3d2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732159605421 2024-11-21T03:26:49,830 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b89772677c24d05aa1272cff1adece0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732159606482 2024-11-21T03:26:49,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742527_1703 (size=12454) 2024-11-21T03:26:49,835 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#C#compaction#594 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:49,835 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c561d2990b644e9f8f44db7c123dafcb is 50, key is test_row_0/C:col10/1732159606487/Put/seqid=0 2024-11-21T03:26:49,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742528_1704 (size=13221) 2024-11-21T03:26:49,839 INFO [Thread-2623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2623. 2024-11-21T03:26:49,839 INFO [Thread-2617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2617. 2024-11-21T03:26:49,839 DEBUG [Thread-2617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:49,839 DEBUG [Thread-2623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:49,839 DEBUG [Thread-2617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:49,839 DEBUG [Thread-2623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:49,839 DEBUG [Thread-2617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:49,839 DEBUG [Thread-2623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:49,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:49,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:49,841 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be8e6cfcf7075b5187319303527b737#A#compaction#591 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:26:49,842 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/400bb57f430a4c5983812d532c74e30b is 175, key is test_row_0/A:col10/1732159606487/Put/seqid=0 2024-11-21T03:26:49,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742529_1705 (size=32175) 2024-11-21T03:26:49,859 INFO [Thread-2621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2621. 2024-11-21T03:26:49,859 INFO [Thread-2619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2619. 2024-11-21T03:26:49,859 DEBUG [Thread-2621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:49,859 DEBUG [Thread-2619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:26:49,859 DEBUG [Thread-2621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:49,859 DEBUG [Thread-2619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:49,859 DEBUG [Thread-2621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:49,859 DEBUG [Thread-2619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:49,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:49,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:50,231 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:50,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cfd7e0912ed94de09f21c4cea9ea4b53_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cfd7e0912ed94de09f21c4cea9ea4b53_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:50,235 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/33f9febb8c6e456989745f1ad5030b86, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:50,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/33f9febb8c6e456989745f1ad5030b86 is 175, key is test_row_0/A:col10/1732159607660/Put/seqid=0 2024-11-21T03:26:50,242 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c561d2990b644e9f8f44db7c123dafcb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c561d2990b644e9f8f44db7c123dafcb 2024-11-21T03:26:50,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742530_1706 (size=31255) 2024-11-21T03:26:50,246 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/C of 0be8e6cfcf7075b5187319303527b737 into c561d2990b644e9f8f44db7c123dafcb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:50,246 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:50,246 INFO [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/C, priority=13, startTime=1732159609411; duration=0sec 2024-11-21T03:26:50,246 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:50,246 DEBUG [RS:0;b69e2a1ee061:45649-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:C 2024-11-21T03:26:50,249 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/33f9febb8c6e456989745f1ad5030b86 2024-11-21T03:26:50,249 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/400bb57f430a4c5983812d532c74e30b as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/400bb57f430a4c5983812d532c74e30b 2024-11-21T03:26:50,252 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be8e6cfcf7075b5187319303527b737/A of 0be8e6cfcf7075b5187319303527b737 into 400bb57f430a4c5983812d532c74e30b(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:26:50,252 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:50,252 INFO [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., storeName=0be8e6cfcf7075b5187319303527b737/A, priority=13, startTime=1732159609411; duration=0sec 2024-11-21T03:26:50,253 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:26:50,253 DEBUG [RS:0;b69e2a1ee061:45649-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be8e6cfcf7075b5187319303527b737:A 2024-11-21T03:26:50,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/84f2082f56664548b065a1fca760107a is 50, key is test_row_0/B:col10/1732159607660/Put/seqid=0 2024-11-21T03:26:50,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742531_1707 (size=12301) 2024-11-21T03:26:50,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/84f2082f56664548b065a1fca760107a 2024-11-21T03:26:50,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c3810e143d0840c18acded3a5c813113 is 50, key is test_row_0/C:col10/1732159607660/Put/seqid=0 2024-11-21T03:26:50,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742532_1708 (size=12301) 2024-11-21T03:26:51,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c3810e143d0840c18acded3a5c813113 2024-11-21T03:26:51,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/33f9febb8c6e456989745f1ad5030b86 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/33f9febb8c6e456989745f1ad5030b86 2024-11-21T03:26:51,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/33f9febb8c6e456989745f1ad5030b86, entries=150, sequenceid=411, filesize=30.5 K 2024-11-21T03:26:51,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/84f2082f56664548b065a1fca760107a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/84f2082f56664548b065a1fca760107a 2024-11-21T03:26:51,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/84f2082f56664548b065a1fca760107a, entries=150, sequenceid=411, filesize=12.0 K 2024-11-21T03:26:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/c3810e143d0840c18acded3a5c813113 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c3810e143d0840c18acded3a5c813113 2024-11-21T03:26:51,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c3810e143d0840c18acded3a5c813113, entries=150, sequenceid=411, filesize=12.0 K 2024-11-21T03:26:51,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for 0be8e6cfcf7075b5187319303527b737 in 1265ms, sequenceid=411, compaction requested=false 2024-11-21T03:26:51,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be8e6cfcf7075b5187319303527b737: 2024-11-21T03:26:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T03:26:51,208 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2630 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2631 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2582 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2614 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2626 2024-11-21T03:26:51,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T03:26:51,208 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:26:51,208 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:26:51,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:51,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:51,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:51,209 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T03:26:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T03:26:51,212 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159611212"}]},"ts":"1732159611212"} 2024-11-21T03:26:51,214 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T03:26:51,214 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T03:26:51,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:26:51,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, UNASSIGN}] 2024-11-21T03:26:51,216 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, UNASSIGN 2024-11-21T03:26:51,217 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=0be8e6cfcf7075b5187319303527b737, regionState=CLOSING, regionLocation=b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:51,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, UNASSIGN because future has completed 2024-11-21T03:26:51,218 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:26:51,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548}] 2024-11-21T03:26:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T03:26:51,370 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:51,370 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:26:51,370 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 0be8e6cfcf7075b5187319303527b737, disabling compactions & flushes 2024-11-21T03:26:51,370 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:51,370 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:51,370 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. after waiting 0 ms 2024-11-21T03:26:51,370 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:51,370 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 0be8e6cfcf7075b5187319303527b737 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-21T03:26:51,374 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112177c788a2c9bb4b459f5f0fffb1619dd2_0be8e6cfcf7075b5187319303527b737 is 50, key is test_row_0/A:col10/1732159609858/Put/seqid=0 2024-11-21T03:26:51,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742533_1709 (size=9914) 2024-11-21T03:26:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T03:26:51,778 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:26:51,780 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112177c788a2c9bb4b459f5f0fffb1619dd2_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112177c788a2c9bb4b459f5f0fffb1619dd2_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:51,781 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/ed9298542fa84a63b6b779626236828a, store: [table=TestAcidGuarantees family=A region=0be8e6cfcf7075b5187319303527b737] 2024-11-21T03:26:51,782 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/ed9298542fa84a63b6b779626236828a is 175, key is test_row_0/A:col10/1732159609858/Put/seqid=0 2024-11-21T03:26:51,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742534_1710 (size=22561) 2024-11-21T03:26:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T03:26:52,185 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/ed9298542fa84a63b6b779626236828a 2024-11-21T03:26:52,190 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/c574102ebc6242e49cafc00ac1698ff1 is 50, key is test_row_0/B:col10/1732159609858/Put/seqid=0 2024-11-21T03:26:52,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742535_1711 (size=9857) 2024-11-21T03:26:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T03:26:52,593 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/c574102ebc6242e49cafc00ac1698ff1 2024-11-21T03:26:52,600 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/f026260498734319a8beaf8f3f06ae85 is 50, key is test_row_0/C:col10/1732159609858/Put/seqid=0 2024-11-21T03:26:52,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742536_1712 (size=9857) 2024-11-21T03:26:52,623 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/f026260498734319a8beaf8f3f06ae85 2024-11-21T03:26:52,628 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/A/ed9298542fa84a63b6b779626236828a as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/ed9298542fa84a63b6b779626236828a 2024-11-21T03:26:52,631 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/ed9298542fa84a63b6b779626236828a, entries=100, sequenceid=421, filesize=22.0 K 2024-11-21T03:26:52,632 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/B/c574102ebc6242e49cafc00ac1698ff1 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/c574102ebc6242e49cafc00ac1698ff1 2024-11-21T03:26:52,634 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/c574102ebc6242e49cafc00ac1698ff1, entries=100, sequenceid=421, filesize=9.6 K 2024-11-21T03:26:52,635 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/.tmp/C/f026260498734319a8beaf8f3f06ae85 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/f026260498734319a8beaf8f3f06ae85 2024-11-21T03:26:52,638 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/f026260498734319a8beaf8f3f06ae85, entries=100, sequenceid=421, filesize=9.6 K 2024-11-21T03:26:52,639 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0be8e6cfcf7075b5187319303527b737 in 1268ms, sequenceid=421, compaction requested=true 2024-11-21T03:26:52,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2d3ea8302c1047918532df45ce8e56c3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5fb13d7c49ca47f5b4ebecad01633e86, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/77be856bfd0d4d4e8740ff55476cd0d7, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/317d4444d8674a8aaa1ac353f338303b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/62e5b970271342d7adee3f24609b633c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0356ebdd74c1486397a6baf071b2c2a8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f235a001d513492eb879b6d59a695f7c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0c3070fb25d84a86bce7abc94dbb9b9b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/bb51372614c6405787e90b784727b755, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/a32feca658d64afbbb9a00c6e20ef142, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f33b997ad91845009304191e0d39e8e5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0be73d429d814e89a0926e45d7d86889, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1db42d0f59874f06b463baa569415ec6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2fc838d1d07b4edea020f147eea33b74, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5432b23832cc45b1baf9f7219480c54f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/ed87dea1c79742b28fd08ad08da00b33, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/c9408f5f452841dd821e2a5bfd676822, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2bf99c1a89a747bc8108435b37c4596e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/95fcb719bbb04a1db125e4d5d81539a2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2627f34811a94b6f884378361d37427d, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0648079ad3704413858eeb12d35923d5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/fdc53eb1459e45d7b54067d89b9fd6a3, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b5fcccd1a1564848a1f43aba88fc891e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/157b528c740c40c98cb848983ff68aba, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1370cc916a1a41daa1a84f0e7d01cb58, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b51f5839d1a340759d1a8b7ffe0b2212, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f13095b4da5244f68f475f1660dc2da1] to archive 2024-11-21T03:26:52,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:26:52,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2d3ea8302c1047918532df45ce8e56c3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2d3ea8302c1047918532df45ce8e56c3 2024-11-21T03:26:52,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5fb13d7c49ca47f5b4ebecad01633e86 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5fb13d7c49ca47f5b4ebecad01633e86 2024-11-21T03:26:52,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/77be856bfd0d4d4e8740ff55476cd0d7 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/77be856bfd0d4d4e8740ff55476cd0d7 2024-11-21T03:26:52,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/317d4444d8674a8aaa1ac353f338303b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/317d4444d8674a8aaa1ac353f338303b 2024-11-21T03:26:52,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/62e5b970271342d7adee3f24609b633c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/62e5b970271342d7adee3f24609b633c 2024-11-21T03:26:52,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0356ebdd74c1486397a6baf071b2c2a8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0356ebdd74c1486397a6baf071b2c2a8 2024-11-21T03:26:52,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f235a001d513492eb879b6d59a695f7c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f235a001d513492eb879b6d59a695f7c 2024-11-21T03:26:52,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0c3070fb25d84a86bce7abc94dbb9b9b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0c3070fb25d84a86bce7abc94dbb9b9b 2024-11-21T03:26:52,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/bb51372614c6405787e90b784727b755 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/bb51372614c6405787e90b784727b755 2024-11-21T03:26:52,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/a32feca658d64afbbb9a00c6e20ef142 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/a32feca658d64afbbb9a00c6e20ef142 2024-11-21T03:26:52,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f33b997ad91845009304191e0d39e8e5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f33b997ad91845009304191e0d39e8e5 2024-11-21T03:26:52,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0be73d429d814e89a0926e45d7d86889 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0be73d429d814e89a0926e45d7d86889 2024-11-21T03:26:52,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1db42d0f59874f06b463baa569415ec6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1db42d0f59874f06b463baa569415ec6 2024-11-21T03:26:52,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2fc838d1d07b4edea020f147eea33b74 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2fc838d1d07b4edea020f147eea33b74 2024-11-21T03:26:52,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5432b23832cc45b1baf9f7219480c54f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/5432b23832cc45b1baf9f7219480c54f 2024-11-21T03:26:52,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/ed87dea1c79742b28fd08ad08da00b33 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/ed87dea1c79742b28fd08ad08da00b33 2024-11-21T03:26:52,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/c9408f5f452841dd821e2a5bfd676822 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/c9408f5f452841dd821e2a5bfd676822 2024-11-21T03:26:52,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2bf99c1a89a747bc8108435b37c4596e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2bf99c1a89a747bc8108435b37c4596e 2024-11-21T03:26:52,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/95fcb719bbb04a1db125e4d5d81539a2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/95fcb719bbb04a1db125e4d5d81539a2 2024-11-21T03:26:52,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2627f34811a94b6f884378361d37427d to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/2627f34811a94b6f884378361d37427d 2024-11-21T03:26:52,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0648079ad3704413858eeb12d35923d5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/0648079ad3704413858eeb12d35923d5 2024-11-21T03:26:52,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/fdc53eb1459e45d7b54067d89b9fd6a3 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/fdc53eb1459e45d7b54067d89b9fd6a3 2024-11-21T03:26:52,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b5fcccd1a1564848a1f43aba88fc891e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b5fcccd1a1564848a1f43aba88fc891e 2024-11-21T03:26:52,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/157b528c740c40c98cb848983ff68aba to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/157b528c740c40c98cb848983ff68aba 2024-11-21T03:26:52,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1370cc916a1a41daa1a84f0e7d01cb58 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/1370cc916a1a41daa1a84f0e7d01cb58 2024-11-21T03:26:52,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b51f5839d1a340759d1a8b7ffe0b2212 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/b51f5839d1a340759d1a8b7ffe0b2212 2024-11-21T03:26:52,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f13095b4da5244f68f475f1660dc2da1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/f13095b4da5244f68f475f1660dc2da1 2024-11-21T03:26:52,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/e82d707d72bb430c84e12b981cb05cd6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/0fbf87bdf07544c2ab1bf0e78eed1abf, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/6bfd16b893e14be8bca290dbe004a087, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/bbeac2fac4f745a1813f7bbb6066781b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/9c31ea85543c4823995b933c8a130c7f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4c4dfd105d2940679329d6b2f4e13bad, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5c078e2ff55d4b3eb5f8cdea443bb384, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/79eef470121e4eea9da5c3c31551071f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/730ad1d1ee8244b581037380d72f702c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/d0891e355ec0432eb46fa134f35d553f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/95e523ac0c3548499723c3b3543cd479, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5f89e8720c714c5ba13fd92e5fe5510e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/261c10f614fa43d79d5df822988a8a7b, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/405e33984b9e47deb27df6163d31aa5e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/a6d96544e04144fc9e1e22a1bbf6bec4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/3db12c4bb56940faa4aac50abcf188ab, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ad14a84922c1459c9af7788a50bbd780, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4f90847d71c74b04b7325fb5b1e6d034, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5de13b3d86be43cab074d156f5ecfc4c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f9e8d412aaa4cdfb820bab493704c7c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2023608fbae941ea8fe51efb62c44fc8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/03d12433a6c64f91b55baf821a421190, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ca23fc01f0fb492595a6af1cadc4f312, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/99eaf0360cea4c4aaf5121dca5e83b8c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2da526dc923c4cd994e576271c912a31, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f58ea91bb3349b8871de78b7ab72391, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/63731c76d0854b278ed5e3cb791030c4] to archive 2024-11-21T03:26:52,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:26:52,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/e82d707d72bb430c84e12b981cb05cd6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/e82d707d72bb430c84e12b981cb05cd6 2024-11-21T03:26:52,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/0fbf87bdf07544c2ab1bf0e78eed1abf to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/0fbf87bdf07544c2ab1bf0e78eed1abf 2024-11-21T03:26:52,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/6bfd16b893e14be8bca290dbe004a087 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/6bfd16b893e14be8bca290dbe004a087 2024-11-21T03:26:52,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/bbeac2fac4f745a1813f7bbb6066781b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/bbeac2fac4f745a1813f7bbb6066781b 2024-11-21T03:26:52,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/9c31ea85543c4823995b933c8a130c7f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/9c31ea85543c4823995b933c8a130c7f 2024-11-21T03:26:52,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4c4dfd105d2940679329d6b2f4e13bad to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4c4dfd105d2940679329d6b2f4e13bad 2024-11-21T03:26:52,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5c078e2ff55d4b3eb5f8cdea443bb384 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5c078e2ff55d4b3eb5f8cdea443bb384 2024-11-21T03:26:52,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/79eef470121e4eea9da5c3c31551071f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/79eef470121e4eea9da5c3c31551071f 2024-11-21T03:26:52,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/730ad1d1ee8244b581037380d72f702c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/730ad1d1ee8244b581037380d72f702c 2024-11-21T03:26:52,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/d0891e355ec0432eb46fa134f35d553f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/d0891e355ec0432eb46fa134f35d553f 2024-11-21T03:26:52,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/95e523ac0c3548499723c3b3543cd479 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/95e523ac0c3548499723c3b3543cd479 2024-11-21T03:26:52,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5f89e8720c714c5ba13fd92e5fe5510e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5f89e8720c714c5ba13fd92e5fe5510e 2024-11-21T03:26:52,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/261c10f614fa43d79d5df822988a8a7b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/261c10f614fa43d79d5df822988a8a7b 2024-11-21T03:26:52,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/405e33984b9e47deb27df6163d31aa5e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/405e33984b9e47deb27df6163d31aa5e 2024-11-21T03:26:52,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/a6d96544e04144fc9e1e22a1bbf6bec4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/a6d96544e04144fc9e1e22a1bbf6bec4 2024-11-21T03:26:52,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/3db12c4bb56940faa4aac50abcf188ab to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/3db12c4bb56940faa4aac50abcf188ab 2024-11-21T03:26:52,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ad14a84922c1459c9af7788a50bbd780 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ad14a84922c1459c9af7788a50bbd780 2024-11-21T03:26:52,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4f90847d71c74b04b7325fb5b1e6d034 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/4f90847d71c74b04b7325fb5b1e6d034 2024-11-21T03:26:52,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5de13b3d86be43cab074d156f5ecfc4c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/5de13b3d86be43cab074d156f5ecfc4c 2024-11-21T03:26:52,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f9e8d412aaa4cdfb820bab493704c7c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f9e8d412aaa4cdfb820bab493704c7c 2024-11-21T03:26:52,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2023608fbae941ea8fe51efb62c44fc8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2023608fbae941ea8fe51efb62c44fc8 2024-11-21T03:26:52,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/03d12433a6c64f91b55baf821a421190 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/03d12433a6c64f91b55baf821a421190 2024-11-21T03:26:52,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ca23fc01f0fb492595a6af1cadc4f312 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/ca23fc01f0fb492595a6af1cadc4f312 2024-11-21T03:26:52,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/99eaf0360cea4c4aaf5121dca5e83b8c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/99eaf0360cea4c4aaf5121dca5e83b8c 2024-11-21T03:26:52,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2da526dc923c4cd994e576271c912a31 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/2da526dc923c4cd994e576271c912a31 2024-11-21T03:26:52,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f58ea91bb3349b8871de78b7ab72391 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/8f58ea91bb3349b8871de78b7ab72391 2024-11-21T03:26:52,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/63731c76d0854b278ed5e3cb791030c4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/63731c76d0854b278ed5e3cb791030c4 2024-11-21T03:26:52,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1c5b2138e854f6c815414d61d51675c, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1e6751ba28343a6ba41e7bdbe7ae085, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/ed21f60fa49046dea058a8c611c5b7c6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d4dbb0911a464ebfa38d69947e395cf4, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/a61665f997b346fc9a503fbfef6e858f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/64deb3ab9bef4d0b81e771a9f60f18f8, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/6416548be1af438b86b9ec4ccca94247, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c0acab149f16493597ec0cae6d0cdf44, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0f10b0f4fc6948d0b86de5327bf530bc, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9c313b11fc6d4898ab4fbb617a056e16, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1677cf8b98049b3ae94a9ce443ad8ce, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c018569554da4850aa3135ca9feebd4f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1af959ce38f84953971740c7fba8be3a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bf8e990e9e4a41e58fde3eab5d750812, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bd65057e6b93403188b6666e65f50b1e, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/5ab55bf5ea404a27a914209a2a66194a, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/e88655ad650e435bbc6c798c2e9aa9c1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/83a627f918ef4667a925d50647b9f8a2, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/7e0c664b94cb4fbf963867f9ae4e2bc6, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9111f4afa2a244a1aa7dfcfb0b010f46, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/238eacb033b44d188666b098558991ca, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/f42e76e3a7264152a91a8d519c4c7de5, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/8b191843fef4458b8fa9bbef3f662851, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9e8f167e3f404cafb682756e21f62e41, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0689c102a771457e8df984ca810b5be1, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/be62cf1ee4b14fa49673dee31dfa3d2f, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1b89772677c24d05aa1272cff1adece0] to archive 2024-11-21T03:26:52,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:26:52,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1c5b2138e854f6c815414d61d51675c to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1c5b2138e854f6c815414d61d51675c 2024-11-21T03:26:52,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1e6751ba28343a6ba41e7bdbe7ae085 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1e6751ba28343a6ba41e7bdbe7ae085 2024-11-21T03:26:52,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/ed21f60fa49046dea058a8c611c5b7c6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/ed21f60fa49046dea058a8c611c5b7c6 2024-11-21T03:26:52,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d4dbb0911a464ebfa38d69947e395cf4 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d4dbb0911a464ebfa38d69947e395cf4 2024-11-21T03:26:52,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/a61665f997b346fc9a503fbfef6e858f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/a61665f997b346fc9a503fbfef6e858f 2024-11-21T03:26:52,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/64deb3ab9bef4d0b81e771a9f60f18f8 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/64deb3ab9bef4d0b81e771a9f60f18f8 2024-11-21T03:26:52,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/6416548be1af438b86b9ec4ccca94247 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/6416548be1af438b86b9ec4ccca94247 2024-11-21T03:26:52,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c0acab149f16493597ec0cae6d0cdf44 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c0acab149f16493597ec0cae6d0cdf44 2024-11-21T03:26:52,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0f10b0f4fc6948d0b86de5327bf530bc to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0f10b0f4fc6948d0b86de5327bf530bc 2024-11-21T03:26:52,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9c313b11fc6d4898ab4fbb617a056e16 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9c313b11fc6d4898ab4fbb617a056e16 2024-11-21T03:26:52,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1677cf8b98049b3ae94a9ce443ad8ce to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/d1677cf8b98049b3ae94a9ce443ad8ce 2024-11-21T03:26:52,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c018569554da4850aa3135ca9feebd4f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c018569554da4850aa3135ca9feebd4f 2024-11-21T03:26:52,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1af959ce38f84953971740c7fba8be3a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1af959ce38f84953971740c7fba8be3a 2024-11-21T03:26:52,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bf8e990e9e4a41e58fde3eab5d750812 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bf8e990e9e4a41e58fde3eab5d750812 2024-11-21T03:26:52,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bd65057e6b93403188b6666e65f50b1e to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/bd65057e6b93403188b6666e65f50b1e 2024-11-21T03:26:52,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/5ab55bf5ea404a27a914209a2a66194a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/5ab55bf5ea404a27a914209a2a66194a 2024-11-21T03:26:52,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/e88655ad650e435bbc6c798c2e9aa9c1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/e88655ad650e435bbc6c798c2e9aa9c1 2024-11-21T03:26:52,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/83a627f918ef4667a925d50647b9f8a2 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/83a627f918ef4667a925d50647b9f8a2 2024-11-21T03:26:52,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/7e0c664b94cb4fbf963867f9ae4e2bc6 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/7e0c664b94cb4fbf963867f9ae4e2bc6 2024-11-21T03:26:52,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9111f4afa2a244a1aa7dfcfb0b010f46 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9111f4afa2a244a1aa7dfcfb0b010f46 2024-11-21T03:26:52,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/238eacb033b44d188666b098558991ca to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/238eacb033b44d188666b098558991ca 2024-11-21T03:26:52,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/f42e76e3a7264152a91a8d519c4c7de5 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/f42e76e3a7264152a91a8d519c4c7de5 2024-11-21T03:26:52,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/8b191843fef4458b8fa9bbef3f662851 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/8b191843fef4458b8fa9bbef3f662851 2024-11-21T03:26:52,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9e8f167e3f404cafb682756e21f62e41 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/9e8f167e3f404cafb682756e21f62e41 2024-11-21T03:26:52,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0689c102a771457e8df984ca810b5be1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/0689c102a771457e8df984ca810b5be1 2024-11-21T03:26:52,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/be62cf1ee4b14fa49673dee31dfa3d2f to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/be62cf1ee4b14fa49673dee31dfa3d2f 2024-11-21T03:26:52,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1b89772677c24d05aa1272cff1adece0 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/1b89772677c24d05aa1272cff1adece0 2024-11-21T03:26:52,789 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/recovered.edits/424.seqid, newMaxSeqId=424, maxSeqId=4 2024-11-21T03:26:52,790 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. 2024-11-21T03:26:52,790 DEBUG [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 0be8e6cfcf7075b5187319303527b737: Waiting for close lock at 1732159611370Running coprocessor pre-close hooks at 1732159611370Disabling compacts and flushes for region at 1732159611370Disabling writes for close at 1732159611370Obtaining lock to block concurrent updates at 1732159611370Preparing flush snapshotting stores in 0be8e6cfcf7075b5187319303527b737 at 1732159611370Finished memstore snapshotting TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732159611370Flushing stores of TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737. at 1732159611371 (+1 ms)Flushing 0be8e6cfcf7075b5187319303527b737/A: creating writer at 1732159611371Flushing mob file 0be8e6cfcf7075b5187319303527b737/A: appending metadata at 1732159611374 (+3 ms)Flushing mob file 0be8e6cfcf7075b5187319303527b737/A: closing flushed file at 1732159611374Flushing 0be8e6cfcf7075b5187319303527b737/A: appending metadata at 1732159611781 (+407 ms)Flushing 0be8e6cfcf7075b5187319303527b737/A: closing flushed file at 1732159611781Flushing 0be8e6cfcf7075b5187319303527b737/B: creating writer at 1732159612188 (+407 ms)Flushing 0be8e6cfcf7075b5187319303527b737/B: appending metadata at 1732159612190 (+2 ms)Flushing 0be8e6cfcf7075b5187319303527b737/B: closing flushed file at 1732159612190Flushing 0be8e6cfcf7075b5187319303527b737/C: creating writer at 1732159612597 (+407 ms)Flushing 0be8e6cfcf7075b5187319303527b737/C: appending metadata at 1732159612599 (+2 ms)Flushing 0be8e6cfcf7075b5187319303527b737/C: closing flushed file at 1732159612599Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f39467d: reopening flushed file at 1732159612627 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2666455f: reopening flushed file at 1732159612631 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6115b409: reopening flushed file at 1732159612634 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0be8e6cfcf7075b5187319303527b737 in 1268ms, sequenceid=421, compaction requested=true at 1732159612639 (+5 ms)Writing region close event to WAL at 1732159612781 (+142 ms)Running coprocessor post-close hooks at 1732159612789 (+8 ms)Closed at 1732159612789 2024-11-21T03:26:52,792 INFO [RS_CLOSE_REGION-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:52,792 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=0be8e6cfcf7075b5187319303527b737, regionState=CLOSED 2024-11-21T03:26:52,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 because future has completed 2024-11-21T03:26:52,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-21T03:26:52,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 0be8e6cfcf7075b5187319303527b737, server=b69e2a1ee061,45649,1732159441548 in 1.5800 sec 2024-11-21T03:26:52,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-21T03:26:52,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be8e6cfcf7075b5187319303527b737, UNASSIGN in 1.5850 sec 2024-11-21T03:26:52,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-21T03:26:52,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5910 sec 2024-11-21T03:26:52,809 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732159612809"}]},"ts":"1732159612809"} 2024-11-21T03:26:52,811 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T03:26:52,811 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T03:26:52,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6030 sec 2024-11-21T03:26:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T03:26:53,358 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:53,358 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T03:26:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:53,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T03:26:53,361 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:53,363 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,365 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C, FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/recovered.edits] 2024-11-21T03:26:53,368 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/33f9febb8c6e456989745f1ad5030b86 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/33f9febb8c6e456989745f1ad5030b86 2024-11-21T03:26:53,369 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/400bb57f430a4c5983812d532c74e30b to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/400bb57f430a4c5983812d532c74e30b 2024-11-21T03:26:53,371 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/ed9298542fa84a63b6b779626236828a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/A/ed9298542fa84a63b6b779626236828a 2024-11-21T03:26:53,373 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/84f2082f56664548b065a1fca760107a to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/84f2082f56664548b065a1fca760107a 2024-11-21T03:26:53,375 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/bc13ccb8954b44cba04892c0234bcca9 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/bc13ccb8954b44cba04892c0234bcca9 2024-11-21T03:26:53,376 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/c574102ebc6242e49cafc00ac1698ff1 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/B/c574102ebc6242e49cafc00ac1698ff1 2024-11-21T03:26:53,378 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c3810e143d0840c18acded3a5c813113 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c3810e143d0840c18acded3a5c813113 2024-11-21T03:26:53,379 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c561d2990b644e9f8f44db7c123dafcb to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/c561d2990b644e9f8f44db7c123dafcb 2024-11-21T03:26:53,380 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/f026260498734319a8beaf8f3f06ae85 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/C/f026260498734319a8beaf8f3f06ae85 2024-11-21T03:26:53,383 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/recovered.edits/424.seqid to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737/recovered.edits/424.seqid 2024-11-21T03:26:53,384 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/default/TestAcidGuarantees/0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,384 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T03:26:53,384 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T03:26:53,385 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T03:26:53,388 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112106e13a0304424b13b1245ebdf6e9f97b_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112106e13a0304424b13b1245ebdf6e9f97b_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,390 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210838fa8210e4470597b8c0b7c9c988a6_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210838fa8210e4470597b8c0b7c9c988a6_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,391 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121124c4dc16df9419f83e988c9b68b260e_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121124c4dc16df9419f83e988c9b68b260e_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,392 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121243dae681d8b42c3a3615f38d7cd5eba_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121243dae681d8b42c3a3615f38d7cd5eba_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,394 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112134b128331a574fa9992684975c4fd8eb_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112134b128331a574fa9992684975c4fd8eb_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,399 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112137df705069e342bcb67cd0275ad4aafb_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112137df705069e342bcb67cd0275ad4aafb_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,400 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213b6b258fc99941bf9daca1dd4c8638c9_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213b6b258fc99941bf9daca1dd4c8638c9_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,401 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112143f223dd48a847f8a03814b9fdaedd23_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112143f223dd48a847f8a03814b9fdaedd23_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,402 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215875d6b76bdb48d88550b6f588445780_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215875d6b76bdb48d88550b6f588445780_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,403 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121604f5f4843ab48d39f3128d945b340b1_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121604f5f4843ab48d39f3128d945b340b1_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,404 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112164bcf112be284787b49183c3c7053319_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112164bcf112be284787b49183c3c7053319_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,404 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216c630cffaab442f4bf9c08c3c0f1e390_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216c630cffaab442f4bf9c08c3c0f1e390_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,405 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112172e6ab479f184a7a9376351f616560a5_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112172e6ab479f184a7a9376351f616560a5_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,406 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112177c788a2c9bb4b459f5f0fffb1619dd2_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112177c788a2c9bb4b459f5f0fffb1619dd2_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,406 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112184a846e958af453896aad5355df6d156_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112184a846e958af453896aad5355df6d156_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,407 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121986eead2503949ef8b61eae4133b9585_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121986eead2503949ef8b61eae4133b9585_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,408 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a856c9db6f1544bb9c7f1a7d86f273ff_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a856c9db6f1544bb9c7f1a7d86f273ff_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,408 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ce56752dcd9042ac91671770eb9075b6_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ce56752dcd9042ac91671770eb9075b6_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,409 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cfd7e0912ed94de09f21c4cea9ea4b53_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cfd7e0912ed94de09f21c4cea9ea4b53_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,410 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e635eda8adc342b0b05c1de553662216_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e635eda8adc342b0b05c1de553662216_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,410 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e7a03338d89b46b8bfe01146d7648993_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e7a03338d89b46b8bfe01146d7648993_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,411 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f3b1a0c023cf4091b739b5eae9cc5f4d_0be8e6cfcf7075b5187319303527b737 to hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f3b1a0c023cf4091b739b5eae9cc5f4d_0be8e6cfcf7075b5187319303527b737 2024-11-21T03:26:53,411 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T03:26:53,413 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:53,414 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T03:26:53,416 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T03:26:53,417 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:53,417 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T03:26:53,417 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732159613417"}]},"ts":"9223372036854775807"} 2024-11-21T03:26:53,419 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T03:26:53,419 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0be8e6cfcf7075b5187319303527b737, NAME => 'TestAcidGuarantees,,1732159586356.0be8e6cfcf7075b5187319303527b737.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T03:26:53,419 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T03:26:53,419 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732159613419"}]},"ts":"9223372036854775807"} 2024-11-21T03:26:53,428 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T03:26:53,429 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:26:53,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-21T03:26:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42023 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T03:26:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T03:26:53,468 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:26:53,478 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 239) - Thread LEAK? -, OpenFileDescriptor=457 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1015 (was 1048), ProcessCount=11 (was 11), AvailableMemoryMB=2620 (was 2765) 2024-11-21T03:26:53,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T03:26:53,479 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:26:53,479 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:26:53,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:53,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:53,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:26:53,479 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T03:26:53,479 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-21T03:26:53,480 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b69e2a1ee061,42023,1732159440464 2024-11-21T03:26:53,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T03:26:53,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T03:26:53,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:26:53,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:26:53,514 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T03:26:53,515 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:26:53,515 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:26:53,516 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:26:53,516 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:26:53,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:53,517 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b69e2a1ee061,45649,1732159441548' ***** 2024-11-21T03:26:53,517 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T03:26:53,517 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T03:26:53,517 INFO [RS:0;b69e2a1ee061:45649 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T03:26:53,517 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T03:26:53,517 INFO [RS:0;b69e2a1ee061:45649 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T03:26:53,518 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(959): stopping server b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:53,518 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T03:26:53,518 INFO [RS:0;b69e2a1ee061:45649 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b69e2a1ee061:45649. 2024-11-21T03:26:53,518 DEBUG [RS:0;b69e2a1ee061:45649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:26:53,518 DEBUG [RS:0;b69e2a1ee061:45649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:53,518 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T03:26:53,518 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T03:26:53,518 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T03:26:53,518 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T03:26:53,519 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T03:26:53,519 DEBUG [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T03:26:53,519 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T03:26:53,519 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T03:26:53,519 DEBUG [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T03:26:53,519 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T03:26:53,519 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T03:26:53,519 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T03:26:53,519 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-21T03:26:53,544 INFO [regionserver/b69e2a1ee061:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:26:53,545 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/info/057df0b574ce4191b1885914fd788b12 is 95, key is TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04./info:/1732159470465/DeleteFamily/seqid=0 2024-11-21T03:26:53,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742537_1713 (size=6514) 2024-11-21T03:26:53,548 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/info/057df0b574ce4191b1885914fd788b12 2024-11-21T03:26:53,568 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/ns/9bb316818ed04fc0b8a0cf8ba52991b2 is 93, key is TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04./ns:/1732159470413/DeleteFamily/seqid=0 2024-11-21T03:26:53,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742538_1714 (size=5906) 2024-11-21T03:26:53,576 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/ns/9bb316818ed04fc0b8a0cf8ba52991b2 2024-11-21T03:26:53,608 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/rep_barrier/1a7fefa2de7b4454aee751dde5ebd506 is 102, key is TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04./rep_barrier:/1732159470413/DeleteFamily/seqid=0 2024-11-21T03:26:53,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742539_1715 (size=6025) 2024-11-21T03:26:53,633 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/rep_barrier/1a7fefa2de7b4454aee751dde5ebd506 2024-11-21T03:26:53,653 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/table/f8549817ebf749468fb27f0c735e0153 is 96, key is TestAcidGuarantees,,1732159445026.aebdff1855164d839f0828203eeecd04./table:/1732159470413/DeleteFamily/seqid=0 2024-11-21T03:26:53,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742540_1716 (size=5936) 2024-11-21T03:26:53,662 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/table/f8549817ebf749468fb27f0c735e0153 2024-11-21T03:26:53,665 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/info/057df0b574ce4191b1885914fd788b12 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/info/057df0b574ce4191b1885914fd788b12 2024-11-21T03:26:53,668 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/info/057df0b574ce4191b1885914fd788b12, entries=12, sequenceid=90, filesize=6.4 K 2024-11-21T03:26:53,671 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/ns/9bb316818ed04fc0b8a0cf8ba52991b2 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/ns/9bb316818ed04fc0b8a0cf8ba52991b2 2024-11-21T03:26:53,674 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/ns/9bb316818ed04fc0b8a0cf8ba52991b2, entries=8, sequenceid=90, filesize=5.8 K 2024-11-21T03:26:53,675 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/rep_barrier/1a7fefa2de7b4454aee751dde5ebd506 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/rep_barrier/1a7fefa2de7b4454aee751dde5ebd506 2024-11-21T03:26:53,678 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/rep_barrier/1a7fefa2de7b4454aee751dde5ebd506, entries=6, sequenceid=90, filesize=5.9 K 2024-11-21T03:26:53,679 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/.tmp/table/f8549817ebf749468fb27f0c735e0153 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/table/f8549817ebf749468fb27f0c735e0153 2024-11-21T03:26:53,682 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/table/f8549817ebf749468fb27f0c735e0153, entries=7, sequenceid=90, filesize=5.8 K 2024-11-21T03:26:53,683 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 164ms, sequenceid=90, compaction requested=false 2024-11-21T03:26:53,688 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-21T03:26:53,689 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T03:26:53,689 INFO [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T03:26:53,689 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732159613519Running coprocessor pre-close hooks at 1732159613519Disabling compacts and flushes for region at 1732159613519Disabling writes for close at 1732159613519Obtaining lock to block concurrent updates at 1732159613519Preparing flush snapshotting stores in 1588230740 at 1732159613519Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732159613520 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732159613520Flushing 1588230740/info: creating writer at 1732159613521 (+1 ms)Flushing 1588230740/info: appending metadata at 1732159613545 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732159613545Flushing 1588230740/ns: creating writer at 1732159613551 (+6 ms)Flushing 1588230740/ns: appending metadata at 1732159613568 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732159613568Flushing 1588230740/rep_barrier: creating writer at 1732159613583 (+15 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732159613608 (+25 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732159613608Flushing 1588230740/table: creating writer at 1732159613636 (+28 ms)Flushing 1588230740/table: appending metadata at 1732159613653 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732159613653Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@698d2c3a: reopening flushed file at 1732159613664 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6eb6328b: reopening flushed file at 1732159613668 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d1dce28: reopening flushed file at 1732159613674 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@106cfe24: reopening flushed file at 1732159613678 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 164ms, sequenceid=90, compaction requested=false at 1732159613683 (+5 ms)Writing region close event to WAL at 1732159613685 (+2 ms)Running coprocessor post-close hooks at 1732159613689 (+4 ms)Closed at 1732159613689 2024-11-21T03:26:53,689 DEBUG [RS_CLOSE_META-regionserver/b69e2a1ee061:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T03:26:53,719 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(976): stopping server b69e2a1ee061,45649,1732159441548; all regions closed. 2024-11-21T03:26:53,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741834_1010 (size=25344) 2024-11-21T03:26:53,725 DEBUG [RS:0;b69e2a1ee061:45649 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/oldWALs 2024-11-21T03:26:53,726 INFO [RS:0;b69e2a1ee061:45649 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b69e2a1ee061%2C45649%2C1732159441548.meta:.meta(num 1732159444455) 2024-11-21T03:26:53,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741833_1009 (size=16383639) 2024-11-21T03:26:53,729 DEBUG [RS:0;b69e2a1ee061:45649 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/oldWALs 2024-11-21T03:26:53,729 INFO [RS:0;b69e2a1ee061:45649 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b69e2a1ee061%2C45649%2C1732159441548:(num 1732159443887) 2024-11-21T03:26:53,729 DEBUG [RS:0;b69e2a1ee061:45649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:26:53,729 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T03:26:53,729 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T03:26:53,729 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.ChoreService(370): Chore service for: regionserver/b69e2a1ee061:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T03:26:53,729 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T03:26:53,729 INFO [regionserver/b69e2a1ee061:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T03:26:53,730 INFO [RS:0;b69e2a1ee061:45649 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45649 2024-11-21T03:26:53,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b69e2a1ee061,45649,1732159441548 2024-11-21T03:26:53,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T03:26:53,735 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T03:26:53,746 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b69e2a1ee061,45649,1732159441548] 2024-11-21T03:26:53,756 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b69e2a1ee061,45649,1732159441548 already deleted, retry=false 2024-11-21T03:26:53,756 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b69e2a1ee061,45649,1732159441548 expired; onlineServers=0 2024-11-21T03:26:53,757 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b69e2a1ee061,42023,1732159440464' ***** 2024-11-21T03:26:53,757 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T03:26:53,757 INFO [M:0;b69e2a1ee061:42023 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T03:26:53,757 INFO [M:0;b69e2a1ee061:42023 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T03:26:53,757 DEBUG [M:0;b69e2a1ee061:42023 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T03:26:53,757 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T03:26:53,757 DEBUG [M:0;b69e2a1ee061:42023 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T03:26:53,757 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster-HFileCleaner.large.0-1732159443487 {}] cleaner.HFileCleaner(306): Exit Thread[master/b69e2a1ee061:0:becomeActiveMaster-HFileCleaner.large.0-1732159443487,5,FailOnTimeoutGroup] 2024-11-21T03:26:53,757 DEBUG [master/b69e2a1ee061:0:becomeActiveMaster-HFileCleaner.small.0-1732159443488 {}] cleaner.HFileCleaner(306): Exit Thread[master/b69e2a1ee061:0:becomeActiveMaster-HFileCleaner.small.0-1732159443488,5,FailOnTimeoutGroup] 2024-11-21T03:26:53,757 INFO [M:0;b69e2a1ee061:42023 {}] hbase.ChoreService(370): Chore service for: master/b69e2a1ee061:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T03:26:53,757 INFO [M:0;b69e2a1ee061:42023 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T03:26:53,758 DEBUG [M:0;b69e2a1ee061:42023 {}] master.HMaster(1795): Stopping service threads 2024-11-21T03:26:53,758 INFO [M:0;b69e2a1ee061:42023 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T03:26:53,758 INFO [M:0;b69e2a1ee061:42023 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T03:26:53,758 ERROR [M:0;b69e2a1ee061:42023 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:41459 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:41459,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-21T03:26:53,759 INFO [M:0;b69e2a1ee061:42023 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T03:26:53,759 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T03:26:53,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T03:26:53,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:26:53,767 DEBUG [M:0;b69e2a1ee061:42023 {}] zookeeper.ZKUtil(347): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T03:26:53,767 WARN [M:0;b69e2a1ee061:42023 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T03:26:53,768 INFO [M:0;b69e2a1ee061:42023 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/.lastflushedseqids 2024-11-21T03:26:53,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742541_1717 (size=72) 2024-11-21T03:26:53,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:26:53,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45649-0x1015b6b7dcc0001, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:26:53,847 INFO [RS:0;b69e2a1ee061:45649 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T03:26:53,848 INFO [RS:0;b69e2a1ee061:45649 {}] regionserver.HRegionServer(1031): Exiting; stopping=b69e2a1ee061,45649,1732159441548; zookeeper connection closed. 2024-11-21T03:26:53,848 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b395bdc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b395bdc 2024-11-21T03:26:53,849 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-21T03:26:54,188 INFO [M:0;b69e2a1ee061:42023 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T03:26:54,188 INFO [M:0;b69e2a1ee061:42023 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T03:26:54,188 DEBUG [M:0;b69e2a1ee061:42023 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T03:26:54,188 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:26:54,188 DEBUG [M:0;b69e2a1ee061:42023 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:26:54,189 DEBUG [M:0;b69e2a1ee061:42023 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-11-21T03:26:54,189 DEBUG [M:0;b69e2a1ee061:42023 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:26:54,189 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=775.59 KB heapSize=954.70 KB 2024-11-21T03:26:54,206 DEBUG [M:0;b69e2a1ee061:42023 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/839b8a77ca064a1e9b421e77a36ac370 is 82, key is hbase:meta,,1/info:regioninfo/1732159444555/Put/seqid=0 2024-11-21T03:26:54,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742542_1718 (size=5672) 2024-11-21T03:26:54,610 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2221 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/839b8a77ca064a1e9b421e77a36ac370 2024-11-21T03:26:54,639 DEBUG [M:0;b69e2a1ee061:42023 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35503623833949abadaf6e2011064e39 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1732159588474/Put/seqid=0 2024-11-21T03:26:54,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742543_1719 (size=43512) 2024-11-21T03:26:55,055 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775.04 KB at sequenceid=2221 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35503623833949abadaf6e2011064e39 2024-11-21T03:26:55,058 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 35503623833949abadaf6e2011064e39 2024-11-21T03:26:55,071 DEBUG [M:0;b69e2a1ee061:42023 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e17aa3af91a44c85bc8f056473186adb is 69, key is b69e2a1ee061,45649,1732159441548/rs:state/1732159443595/Put/seqid=0 2024-11-21T03:26:55,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073742544_1720 (size=5156) 2024-11-21T03:26:55,474 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2221 (bloomFilter=true), to=hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e17aa3af91a44c85bc8f056473186adb 2024-11-21T03:26:55,478 DEBUG [M:0;b69e2a1ee061:42023 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/839b8a77ca064a1e9b421e77a36ac370 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/839b8a77ca064a1e9b421e77a36ac370 2024-11-21T03:26:55,481 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/839b8a77ca064a1e9b421e77a36ac370, entries=8, sequenceid=2221, filesize=5.5 K 2024-11-21T03:26:55,481 DEBUG [M:0;b69e2a1ee061:42023 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35503623833949abadaf6e2011064e39 as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/35503623833949abadaf6e2011064e39 2024-11-21T03:26:55,484 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 35503623833949abadaf6e2011064e39 2024-11-21T03:26:55,484 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/35503623833949abadaf6e2011064e39, entries=166, sequenceid=2221, filesize=42.5 K 2024-11-21T03:26:55,485 DEBUG [M:0;b69e2a1ee061:42023 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e17aa3af91a44c85bc8f056473186adb as hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e17aa3af91a44c85bc8f056473186adb 2024-11-21T03:26:55,487 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41459/user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e17aa3af91a44c85bc8f056473186adb, entries=1, sequenceid=2221, filesize=5.0 K 2024-11-21T03:26:55,488 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.HRegion(3140): Finished flush of dataSize ~775.59 KB/794207, heapSize ~954.41 KB/977312, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2221, compaction requested=false 2024-11-21T03:26:55,492 INFO [M:0;b69e2a1ee061:42023 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:26:55,492 DEBUG [M:0;b69e2a1ee061:42023 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732159614188Disabling compacts and flushes for region at 1732159614188Disabling writes for close at 1732159614189 (+1 ms)Obtaining lock to block concurrent updates at 1732159614189Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732159614189Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=794207, getHeapSize=977552, getOffHeapSize=0, getCellsCount=2412 at 1732159614189Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732159614189Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732159614190 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732159614206 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732159614206Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732159614614 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732159614638 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732159614638Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732159615058 (+420 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732159615070 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732159615071 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6faf9126: reopening flushed file at 1732159615477 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@442a387: reopening flushed file at 1732159615481 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c524c8d: reopening flushed file at 1732159615484 (+3 ms)Finished flush of dataSize ~775.59 KB/794207, heapSize ~954.41 KB/977312, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2221, compaction requested=false at 1732159615488 (+4 ms)Writing region close event to WAL at 1732159615492 (+4 ms)Closed at 1732159615492 2024-11-21T03:26:55,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44993 is added to blk_1073741830_1006 (size=939755) 2024-11-21T03:26:55,495 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/27acfbd3-b964-0e7f-8cb4-aa0b9681b2c1/MasterData/WALs/b69e2a1ee061,42023,1732159440464/b69e2a1ee061%2C42023%2C1732159440464.1732159442429 not finished, retry = 0 2024-11-21T03:26:55,596 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T03:26:55,596 INFO [M:0;b69e2a1ee061:42023 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T03:26:55,596 INFO [M:0;b69e2a1ee061:42023 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42023 2024-11-21T03:26:55,596 INFO [M:0;b69e2a1ee061:42023 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T03:26:55,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:26:55,709 INFO [M:0;b69e2a1ee061:42023 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T03:26:55,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42023-0x1015b6b7dcc0000, quorum=127.0.0.1:61758, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T03:26:55,714 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T03:26:55,718 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T03:26:55,718 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T03:26:55,718 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T03:26:55,718 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/hadoop.log.dir/,STOPPED} 2024-11-21T03:26:55,722 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T03:26:55,722 WARN [BP-547451667-172.17.0.2-1732159435428 heartbeating to localhost/127.0.0.1:41459 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T03:26:55,722 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T03:26:55,722 WARN [BP-547451667-172.17.0.2-1732159435428 heartbeating to localhost/127.0.0.1:41459 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-547451667-172.17.0.2-1732159435428 (Datanode Uuid 338cfed7-1fb1-4a89-94ce-02f4b7325128) service to localhost/127.0.0.1:41459 2024-11-21T03:26:55,725 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/cluster_0b1aaf4b-5e34-6dbd-0179-21f4d40d50a7/data/data1/current/BP-547451667-172.17.0.2-1732159435428 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T03:26:55,725 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/cluster_0b1aaf4b-5e34-6dbd-0179-21f4d40d50a7/data/data2/current/BP-547451667-172.17.0.2-1732159435428 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T03:26:55,726 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T03:26:55,735 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T03:26:55,736 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T03:26:55,736 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T03:26:55,736 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T03:26:55,736 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af46e8c-3f9f-096d-28ae-3f511b658f00/hadoop.log.dir/,STOPPED} 2024-11-21T03:26:55,752 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T03:26:55,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down